From 634604d0b5908b55706b7a0931a256dead75029d Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Tue, 14 Aug 2018 16:59:41 -0500 Subject: [PATCH] Combine price feed transformers - fetches logs from all three price feeds in one query - assumes eth/usd price feed will be updated to include LogValue event - updates transformers to run separate from header sync --- README.md | 23 +-- cmd/backfillMakerLogs.go | 3 +- cmd/lightSync.go | 6 +- cmd/root.go | 12 -- cmd/syncPriceFeeds.go | 88 ---------- .../1532533671_create_peps_table.down.sql | 2 - .../1532533671_create_peps_table.up.sql | 9 -- .../1532623709_create_pips_table.down.sql | 1 - .../1532623709_create_pips_table.up.sql | 9 -- .../1532635100_create_reps_table.down.sql | 1 - .../1532635100_create_reps_table.up.sql | 9 -- ...34275305_create_price_feeds_table.down.sql | 1 + ...1534275305_create_price_feeds_table.up.sql | 12 ++ db/schema.sql | 136 +++------------- environments/infura.toml | 5 +- environments/private.toml | 3 - environments/public.toml.example | 3 - .../repositories/header_repository.go | 1 + pkg/fakes/mock_transformer.go | 44 ----- pkg/history/header_validator.go | 7 +- pkg/history/header_validator_test.go | 18 +-- pkg/history/populate_headers.go | 15 +- pkg/history/populate_headers_test.go | 52 +----- pkg/transformers/flip_kick/config.go | 12 +- pkg/transformers/flip_kick/transformer.go | 10 +- .../flip_kick/transformer_test.go | 11 +- pkg/transformers/frob/config.go | 12 +- pkg/transformers/frob/transformer.go | 10 +- pkg/transformers/frob/transformer_test.go | 5 +- pkg/transformers/price_feeds/config.go | 37 +++++ pkg/transformers/price_feeds/constants.go | 22 ++- pkg/transformers/price_feeds/converter.go | 32 ++++ .../price_feeds/converter_test.go | 57 +++++++ pkg/transformers/price_feeds/fetcher.go | 54 +++++++ pkg/transformers/price_feeds/fetcher_test.go | 64 ++++++++ pkg/transformers/price_feeds/pep/fetcher.go | 53 ------- .../price_feeds/pep/fetcher_test.go | 74 --------- .../price_feeds/pep/pep_suite_test.go | 13 -- .../price_feeds/pep/repository.go | 25 --- .../price_feeds/pep/repository_test.go | 51 ------ .../price_feeds/pep/transformer.go | 43 ----- .../price_feeds/pep/transformer_test.go | 46 ------ pkg/transformers/price_feeds/pip/fetcher.go | 77 --------- .../price_feeds/pip/fetcher_test.go | 61 ------- .../price_feeds/pip/pip_suite_test.go | 13 -- .../price_feeds/pip/repository.go | 25 --- .../price_feeds/pip/repository_test.go | 51 ------ .../price_feeds/pip/transformer.go | 43 ----- .../price_feeds/pip/transformer_test.go | 45 ------ .../price_feeds/price_feeds_suite_test.go | 27 ++++ pkg/transformers/price_feeds/price_update.go | 33 +++- pkg/transformers/price_feeds/rep/fetcher.go | 51 ------ .../price_feeds/rep/fetcher_test.go | 72 --------- .../price_feeds/rep/rep_suite_test.go | 13 -- .../price_feeds/rep/repository.go | 25 --- .../price_feeds/rep/repository_test.go | 50 ------ .../price_feeds/rep/transformer.go | 43 ----- .../price_feeds/rep/transformer_test.go | 46 ------ pkg/transformers/price_feeds/repository.go | 56 +++++++ .../price_feeds/repository_test.go | 147 +++++++++++++++++ pkg/transformers/price_feeds/transformer.go | 65 ++++++++ .../price_feeds/transformer_test.go | 150 ++++++++++++++++++ pkg/transformers/shared/transformer.go | 8 + .../test_data/mocks/price_feeds/fetcher.go | 29 ++++ .../test_data/mocks/price_feeds/repository.go | 49 ++++++ pkg/transformers/transformer.go | 9 -- pkg/transformers/transformers.go | 4 + test_config/test_config.go | 3 - 68 files changed, 902 insertions(+), 1384 deletions(-) delete mode 100644 cmd/syncPriceFeeds.go delete mode 100644 db/migrations/1532533671_create_peps_table.down.sql delete mode 100644 db/migrations/1532533671_create_peps_table.up.sql delete mode 100644 db/migrations/1532623709_create_pips_table.down.sql delete mode 100644 db/migrations/1532623709_create_pips_table.up.sql delete mode 100644 db/migrations/1532635100_create_reps_table.down.sql delete mode 100644 db/migrations/1532635100_create_reps_table.up.sql create mode 100644 db/migrations/1534275305_create_price_feeds_table.down.sql create mode 100644 db/migrations/1534275305_create_price_feeds_table.up.sql delete mode 100644 pkg/fakes/mock_transformer.go create mode 100644 pkg/transformers/price_feeds/config.go create mode 100644 pkg/transformers/price_feeds/converter.go create mode 100644 pkg/transformers/price_feeds/converter_test.go create mode 100644 pkg/transformers/price_feeds/fetcher.go create mode 100644 pkg/transformers/price_feeds/fetcher_test.go delete mode 100644 pkg/transformers/price_feeds/pep/fetcher.go delete mode 100644 pkg/transformers/price_feeds/pep/fetcher_test.go delete mode 100644 pkg/transformers/price_feeds/pep/pep_suite_test.go delete mode 100644 pkg/transformers/price_feeds/pep/repository.go delete mode 100644 pkg/transformers/price_feeds/pep/repository_test.go delete mode 100644 pkg/transformers/price_feeds/pep/transformer.go delete mode 100644 pkg/transformers/price_feeds/pep/transformer_test.go delete mode 100644 pkg/transformers/price_feeds/pip/fetcher.go delete mode 100644 pkg/transformers/price_feeds/pip/fetcher_test.go delete mode 100644 pkg/transformers/price_feeds/pip/pip_suite_test.go delete mode 100644 pkg/transformers/price_feeds/pip/repository.go delete mode 100644 pkg/transformers/price_feeds/pip/repository_test.go delete mode 100644 pkg/transformers/price_feeds/pip/transformer.go delete mode 100644 pkg/transformers/price_feeds/pip/transformer_test.go create mode 100644 pkg/transformers/price_feeds/price_feeds_suite_test.go delete mode 100644 pkg/transformers/price_feeds/rep/fetcher.go delete mode 100644 pkg/transformers/price_feeds/rep/fetcher_test.go delete mode 100644 pkg/transformers/price_feeds/rep/rep_suite_test.go delete mode 100644 pkg/transformers/price_feeds/rep/repository.go delete mode 100644 pkg/transformers/price_feeds/rep/repository_test.go delete mode 100644 pkg/transformers/price_feeds/rep/transformer.go delete mode 100644 pkg/transformers/price_feeds/rep/transformer_test.go create mode 100644 pkg/transformers/price_feeds/repository.go create mode 100644 pkg/transformers/price_feeds/repository_test.go create mode 100644 pkg/transformers/price_feeds/transformer.go create mode 100644 pkg/transformers/price_feeds/transformer_test.go create mode 100644 pkg/transformers/test_data/mocks/price_feeds/fetcher.go create mode 100644 pkg/transformers/test_data/mocks/price_feeds/repository.go delete mode 100644 pkg/transformers/transformer.go diff --git a/README.md b/README.md index a60426d3..d00eccdc 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,7 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli ## Configuration - To use a local Ethereum node, copy `environments/public.toml.example` to - `environments/public.toml` and update the `ipcPath`, `levelDbPath`, - `pipContractAddress`, `pepContractAddress`, and `repContractAddress`. + `environments/public.toml` and update the `ipcPath` and `levelDbPath`. - `ipcPath` should match the local node's IPC filepath: - when using geth: - The IPC file is called `geth.ipc`. @@ -52,10 +51,7 @@ Vulcanize DB is a set of tools that make it easier for developers to write appli - Linux: `$HOME/.ethereum/geth/chaindata` - `levelDbPath` is irrelevant (and `coldImport` is currently unavailable) if only running parity. - - `pepContractAddress`, `pipContractAddress`, and `repContractAddress` should match that medianizer - addresses for each pair on the chain you're tracking. See https://makerdao.com/feeds/ - -- See `environments/infura.toml` to configure commands to run against infura, if a local node is unavailable +- See `environments/infura.toml` to configure commands to run against infura, if a local node is unavailable. - Copy `environments/local.toml.example` to `environments/local.toml` to configure commands to run against a local node such as [Ganache](https://truffleframework.com/ganache) or [ganache-cli](https://github.com/trufflesuite/ganache-clihttps://github.com/trufflesuite/ganache-cli). ## Start syncing with postgres @@ -82,22 +78,17 @@ This command is useful when you want a minimal baseline from which to track targ 1. In a separate terminal start VulcanizeDB: - `./vulcanizedb lightSync --config --starting-block-number ` -## Backfill Auction event logs from light sync -Backfills auction event logs from the configured Ethereum node based on the populated block headers. +## Backfill Maker event logs from light sync +Backfills Maker event logs from the configured Ethereum node based on the populated block headers. +This includes logs related to auctions, multi-collateral dai, and price feeds. This command requires that a light sync (see command above) has previously been run. -_Since auction contracts have not yet been deployed, this command will need to be run a local blockchain at the moment. As such, a new environment file will need to be added. See `environments/local.toml.example`._ +_Since auction/mcd contracts have not yet been deployed, this command will need to be run a local blockchain at the moment. As such, a new environment file will need to be added. See `environments/local.toml.example`._ 1. Start Ethereum node 1. In a separate terminal run the backfill command: - - `./vulcanizedb backfillAuctionLogs --config ` + - `./vulcanizedb backfillMakerLogs --config ` -## Sync in light mode with MakerDAO price feeds -Sync VulcanizeDB with the configured Ethereum node, populating block headers as well as price feeds for MKR/USD, ETH/USD, and REP/USD. -1. Start Ethereum node -1. In a separate terminal window start VulcanizeDB - - `./vulcanizedb syncPriceFeeds --config --starting-block-number ` - ## Start full environment in docker by single command ### Geth Rinkeby diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index f64be700..69aab17e 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -35,7 +35,8 @@ var backfillMakerLogsCmd = &cobra.Command{ Use: "backfillMakerLogs", Short: "Backfill Maker event logs", Long: `Backfills Maker event logs based on previously populated block Header records. -This currently includes logs related to Multi-collateral Dai (frob) and Auctions (flip-kick). +This currently includes logs related to Multi-collateral Dai (frob), Auctions (flip-kick), +and Price Feeds (ETH/USD, MKR/USD, and REP/USD - LogValue). vulcanize backfillMakerLogs --config environments/local.toml diff --git a/cmd/lightSync.go b/cmd/lightSync.go index 0c91b57b..f6ff2679 100644 --- a/cmd/lightSync.go +++ b/cmd/lightSync.go @@ -31,7 +31,6 @@ import ( vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/history" - "github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/utils" ) @@ -65,8 +64,7 @@ func init() { } func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) { - emptyTransformers := []transformers.Transformer{} - populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber, emptyTransformers) + populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber) if err != nil { log.Fatal("Error populating headers: ", err) } @@ -81,7 +79,7 @@ func lightSync() { db := utils.LoadPostgres(databaseConfig, blockChain.Node()) headerRepository := repositories.NewHeaderRepository(&db) - validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow, []transformers.Transformer{}) + validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow) missingBlocksPopulated := make(chan int) go backFillAllHeaders(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber) diff --git a/cmd/root.go b/cmd/root.go index ea51e448..c0336edc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -29,9 +29,6 @@ var ( databaseConfig config.Database ipc string levelDbPath string - pepContractAddress string - pipContractAddress string - repContractAddress string startingBlockNumber int64 syncAll bool endingBlockNumber int64 @@ -52,9 +49,6 @@ func Execute() { func database(cmd *cobra.Command, args []string) { ipc = viper.GetString("client.ipcpath") levelDbPath = viper.GetString("client.leveldbpath") - pepContractAddress = viper.GetString("client.pepcontractaddress") - pipContractAddress = viper.GetString("client.pipcontractaddress") - repContractAddress = viper.GetString("client.repcontractaddress") databaseConfig = config.Database{ Name: viper.GetString("database.name"), Hostname: viper.GetString("database.hostname"), @@ -76,9 +70,6 @@ func init() { rootCmd.PersistentFlags().String("database-password", "", "database password") rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata") - rootCmd.PersistentFlags().String("client-pepContractAddress", "", "mkr/usd price feed contract address") - rootCmd.PersistentFlags().String("client-pipContractAddress", "", "eth/usd price feed contract address") - rootCmd.PersistentFlags().String("client-repContractAddress", "", "rep/usd price feed contract address") viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) @@ -87,9 +78,6 @@ func init() { viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password")) viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath")) viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath")) - viper.BindPFlag("client.pepContractAddress", rootCmd.PersistentFlags().Lookup("client-pepContractAddress")) - viper.BindPFlag("client.pipContractAddress", rootCmd.PersistentFlags().Lookup("client-pipContractAddress")) - viper.BindPFlag("client.repContractAddress", rootCmd.PersistentFlags().Lookup("client-repContractAddress")) } func initConfig() { diff --git a/cmd/syncPriceFeeds.go b/cmd/syncPriceFeeds.go deleted file mode 100644 index 32edaf3f..00000000 --- a/cmd/syncPriceFeeds.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright © 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "log" - "os" - "time" - - "github.com/spf13/cobra" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/history" - "github.com/vulcanize/vulcanizedb/pkg/transformers" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep" - "github.com/vulcanize/vulcanizedb/utils" -) - -// syncPriceFeedsCmd represents the syncPriceFeeds command -var syncPriceFeedsCmd = &cobra.Command{ - Use: "syncPriceFeeds", - Short: "Sync block headers with price feed data", - Long: `Sync Ethereum block headers and price feed data. For example: - -./vulcanizedb syncPriceFeeds --config --starting-block-number - -Price feed data will be updated when price feed contracts log value events.`, - Run: func(cmd *cobra.Command, args []string) { - syncPriceFeeds() - }, -} - -func init() { - rootCmd.AddCommand(syncPriceFeedsCmd) - syncPriceFeedsCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "block number at which to start tracking price feeds") -} - -func backFillPriceFeeds(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64, transformers []transformers.Transformer) { - populated, err := history.PopulateMissingHeaders(blockchain, headerRepository, startingBlockNumber, transformers) - if err != nil { - log.Fatal("Error populating headers: ", err) - } - missingBlocksPopulated <- populated -} - -func syncPriceFeeds() { - ticker := time.NewTicker(pollingInterval) - defer ticker.Stop() - blockChain := getBlockChain() - validateArgs(blockChain) - db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - - transformers := []transformers.Transformer{ - pep.NewPepTransformer(blockChain, &db, pepContractAddress), - pip.NewPipTransformer(blockChain, &db, pipContractAddress), - rep.NewRepTransformer(blockChain, &db, repContractAddress), - } - headerRepository := repositories.NewHeaderRepository(&db) - missingBlocksPopulated := make(chan int) - validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow, transformers) - go backFillPriceFeeds(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber, transformers) - - for { - select { - case <-ticker.C: - window := validator.ValidateHeaders() - window.Log(os.Stdout) - case <-missingBlocksPopulated: - go backFillPriceFeeds(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber, transformers) - } - } -} diff --git a/db/migrations/1532533671_create_peps_table.down.sql b/db/migrations/1532533671_create_peps_table.down.sql deleted file mode 100644 index 2e0b3fd4..00000000 --- a/db/migrations/1532533671_create_peps_table.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE maker.peps; -DROP SCHEMA maker; diff --git a/db/migrations/1532533671_create_peps_table.up.sql b/db/migrations/1532533671_create_peps_table.up.sql deleted file mode 100644 index 7dbf11c7..00000000 --- a/db/migrations/1532533671_create_peps_table.up.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE maker.peps ( - id SERIAL PRIMARY KEY, - block_number BIGINT NOT NULL, - header_id INTEGER NOT NULL, - usd_value NUMERIC, - CONSTRAINT headers_fk FOREIGN KEY (header_id) - REFERENCES headers (id) - ON DELETE CASCADE -); diff --git a/db/migrations/1532623709_create_pips_table.down.sql b/db/migrations/1532623709_create_pips_table.down.sql deleted file mode 100644 index cd5c39b5..00000000 --- a/db/migrations/1532623709_create_pips_table.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE maker.pips; \ No newline at end of file diff --git a/db/migrations/1532623709_create_pips_table.up.sql b/db/migrations/1532623709_create_pips_table.up.sql deleted file mode 100644 index a957e825..00000000 --- a/db/migrations/1532623709_create_pips_table.up.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE maker.pips ( - id SERIAL PRIMARY KEY, - block_number BIGINT NOT NULL, - header_id INTEGER NOT NULL, - usd_value NUMERIC, - CONSTRAINT headers_fk FOREIGN KEY (header_id) - REFERENCES headers (id) - ON DELETE CASCADE -); \ No newline at end of file diff --git a/db/migrations/1532635100_create_reps_table.down.sql b/db/migrations/1532635100_create_reps_table.down.sql deleted file mode 100644 index 5d82e3dc..00000000 --- a/db/migrations/1532635100_create_reps_table.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE maker.reps; \ No newline at end of file diff --git a/db/migrations/1532635100_create_reps_table.up.sql b/db/migrations/1532635100_create_reps_table.up.sql deleted file mode 100644 index 7f79e98f..00000000 --- a/db/migrations/1532635100_create_reps_table.up.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE maker.reps ( - id SERIAL PRIMARY KEY, - block_number BIGINT NOT NULL, - header_id INTEGER NOT NULL, - usd_value NUMERIC, - CONSTRAINT headers_fk FOREIGN KEY (header_id) - REFERENCES headers (id) - ON DELETE CASCADE -); \ No newline at end of file diff --git a/db/migrations/1534275305_create_price_feeds_table.down.sql b/db/migrations/1534275305_create_price_feeds_table.down.sql new file mode 100644 index 00000000..98b365c6 --- /dev/null +++ b/db/migrations/1534275305_create_price_feeds_table.down.sql @@ -0,0 +1 @@ +DROP TABLE maker.price_feeds; \ No newline at end of file diff --git a/db/migrations/1534275305_create_price_feeds_table.up.sql b/db/migrations/1534275305_create_price_feeds_table.up.sql new file mode 100644 index 00000000..a6b6f9a0 --- /dev/null +++ b/db/migrations/1534275305_create_price_feeds_table.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE maker.price_feeds ( + id SERIAL PRIMARY KEY, + block_number BIGINT NOT NULL, + header_id INTEGER NOT NULL, + medianizer_address bytea, + tx_idx INTEGER NOT NULL, + usd_value NUMERIC, + UNIQUE (header_id, medianizer_address, tx_idx), + CONSTRAINT headers_fk FOREIGN KEY (header_id) + REFERENCES headers (id) + ON DELETE CASCADE +); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index bfa6a4ca..3b5fb120 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -120,22 +120,24 @@ ALTER SEQUENCE maker.frob_id_seq OWNED BY maker.frob.id; -- --- Name: peps; Type: TABLE; Schema: maker; Owner: - +-- Name: price_feeds; Type: TABLE; Schema: maker; Owner: - -- -CREATE TABLE maker.peps ( +CREATE TABLE maker.price_feeds ( id integer NOT NULL, block_number bigint NOT NULL, header_id integer NOT NULL, + medianizer_address bytea, + tx_idx integer NOT NULL, usd_value numeric ); -- --- Name: peps_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- Name: price_feeds_id_seq; Type: SEQUENCE; Schema: maker; Owner: - -- -CREATE SEQUENCE maker.peps_id_seq +CREATE SEQUENCE maker.price_feeds_id_seq AS integer START WITH 1 INCREMENT BY 1 @@ -145,74 +147,10 @@ CREATE SEQUENCE maker.peps_id_seq -- --- Name: peps_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- Name: price_feeds_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - -- -ALTER SEQUENCE maker.peps_id_seq OWNED BY maker.peps.id; - - --- --- Name: pips; Type: TABLE; Schema: maker; Owner: - --- - -CREATE TABLE maker.pips ( - id integer NOT NULL, - block_number bigint NOT NULL, - header_id integer NOT NULL, - usd_value numeric -); - - --- --- Name: pips_id_seq; Type: SEQUENCE; Schema: maker; Owner: - --- - -CREATE SEQUENCE maker.pips_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: pips_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - --- - -ALTER SEQUENCE maker.pips_id_seq OWNED BY maker.pips.id; - - --- --- Name: reps; Type: TABLE; Schema: maker; Owner: - --- - -CREATE TABLE maker.reps ( - id integer NOT NULL, - block_number bigint NOT NULL, - header_id integer NOT NULL, - usd_value numeric -); - - --- --- Name: reps_id_seq; Type: SEQUENCE; Schema: maker; Owner: - --- - -CREATE SEQUENCE maker.reps_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: reps_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - --- - -ALTER SEQUENCE maker.reps_id_seq OWNED BY maker.reps.id; +ALTER SEQUENCE maker.price_feeds_id_seq OWNED BY maker.price_feeds.id; -- @@ -602,24 +540,10 @@ ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_s -- --- Name: peps id; Type: DEFAULT; Schema: maker; Owner: - +-- Name: price_feeds id; Type: DEFAULT; Schema: maker; Owner: - -- -ALTER TABLE ONLY maker.peps ALTER COLUMN id SET DEFAULT nextval('maker.peps_id_seq'::regclass); - - --- --- Name: pips id; Type: DEFAULT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.pips ALTER COLUMN id SET DEFAULT nextval('maker.pips_id_seq'::regclass); - - --- --- Name: reps id; Type: DEFAULT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.reps ALTER COLUMN id SET DEFAULT nextval('maker.reps_id_seq'::regclass); +ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.price_feeds_id_seq'::regclass); -- @@ -718,27 +642,19 @@ ALTER TABLE ONLY maker.frob -- --- Name: peps peps_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- Name: price_feeds price_feeds_header_id_medianizer_address_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- -ALTER TABLE ONLY maker.peps - ADD CONSTRAINT peps_pkey PRIMARY KEY (id); +ALTER TABLE ONLY maker.price_feeds + ADD CONSTRAINT price_feeds_header_id_medianizer_address_tx_idx_key UNIQUE (header_id, medianizer_address, tx_idx); -- --- Name: pips pips_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- Name: price_feeds price_feeds_pkey; Type: CONSTRAINT; Schema: maker; Owner: - -- -ALTER TABLE ONLY maker.pips - ADD CONSTRAINT pips_pkey PRIMARY KEY (id); - - --- --- Name: reps reps_pkey; Type: CONSTRAINT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.reps - ADD CONSTRAINT reps_pkey PRIMARY KEY (id); +ALTER TABLE ONLY maker.price_feeds + ADD CONSTRAINT price_feeds_pkey PRIMARY KEY (id); -- @@ -889,26 +805,10 @@ ALTER TABLE ONLY maker.frob -- --- Name: peps headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- Name: price_feeds headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: - -- -ALTER TABLE ONLY maker.peps - ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; - - --- --- Name: pips headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.pips - ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; - - --- --- Name: reps headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: - --- - -ALTER TABLE ONLY maker.reps +ALTER TABLE ONLY maker.price_feeds ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; diff --git a/environments/infura.toml b/environments/infura.toml index 30db3365..bd273b20 100644 --- a/environments/infura.toml +++ b/environments/infura.toml @@ -1,10 +1,7 @@ [database] -name = "vulcanize_public" +name = "vulcanize_private" hostname = "localhost" port = 5432 [client] ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" -pepContractAddress = "0x99041F808D598B782D5a3e498681C2452A31da08" -pipContractAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B" -repContractAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C" diff --git a/environments/private.toml b/environments/private.toml index 2193e44f..344b7a94 100644 --- a/environments/private.toml +++ b/environments/private.toml @@ -5,6 +5,3 @@ port = 5432 [client] ipcPath = "http://127.0.0.1:7545" -pepContractAddress = "0x99041F808D598B782D5a3e498681C2452A31da08" -pipContractAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B" -repContractAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C" diff --git a/environments/public.toml.example b/environments/public.toml.example index c92e03a9..ac9dbf85 100644 --- a/environments/public.toml.example +++ b/environments/public.toml.example @@ -6,6 +6,3 @@ port = 5432 [client] ipcPath = levelDbPath = -pepContractAddress = -pipContractAddress = -repContractAddress = \ No newline at end of file diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index 981888ba..b9743be8 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -3,6 +3,7 @@ package repositories import ( "database/sql" "errors" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) diff --git a/pkg/fakes/mock_transformer.go b/pkg/fakes/mock_transformer.go deleted file mode 100644 index 23a40e37..00000000 --- a/pkg/fakes/mock_transformer.go +++ /dev/null @@ -1,44 +0,0 @@ -package fakes - -import ( - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/pkg/core" -) - -type MockTransformer struct { - passedHeader core.Header - passedHeaderID int64 - executeCalled bool - executeErr error -} - -func NewMockTransformer() *MockTransformer { - return &MockTransformer{ - passedHeader: core.Header{}, - passedHeaderID: 0, - executeCalled: false, - executeErr: nil, - } -} - -func (transformer *MockTransformer) SetExecuteErr(err error) { - transformer.executeErr = err -} - -func (transformer *MockTransformer) Execute(header core.Header, headerID int64) error { - transformer.executeCalled = true - transformer.passedHeader = header - transformer.passedHeaderID = headerID - return transformer.executeErr -} - -func (transformer *MockTransformer) AssertExecuteCalledWith(header core.Header, headerID int64) { - Expect(transformer.executeCalled).To(BeTrue()) - Expect(header).To(Equal(transformer.passedHeader)) - Expect(headerID).To(Equal(transformer.passedHeaderID)) -} - -func (tranformer *MockTransformer) AssertExecuteNotCalled() { - Expect(tranformer.executeCalled).To(BeFalse()) -} diff --git a/pkg/history/header_validator.go b/pkg/history/header_validator.go index add52dc9..b733aaee 100644 --- a/pkg/history/header_validator.go +++ b/pkg/history/header_validator.go @@ -3,28 +3,25 @@ package history import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" - "github.com/vulcanize/vulcanizedb/pkg/transformers" ) type HeaderValidator struct { blockChain core.BlockChain headerRepository datastore.HeaderRepository windowSize int - transformers []transformers.Transformer } -func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int, transformers []transformers.Transformer) HeaderValidator { +func NewHeaderValidator(blockChain core.BlockChain, repository datastore.HeaderRepository, windowSize int) HeaderValidator { return HeaderValidator{ blockChain: blockChain, headerRepository: repository, windowSize: windowSize, - transformers: transformers, } } func (validator HeaderValidator) ValidateHeaders() ValidationWindow { window := MakeValidationWindow(validator.blockChain, validator.windowSize) blockNumbers := MakeRange(window.LowerBound, window.UpperBound) - RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers, validator.transformers) + RetrieveAndUpdateHeaders(validator.blockChain, validator.headerRepository, blockNumbers) return window } diff --git a/pkg/history/header_validator_test.go b/pkg/history/header_validator_test.go index fdc8fb08..379f6739 100644 --- a/pkg/history/header_validator_test.go +++ b/pkg/history/header_validator_test.go @@ -2,10 +2,8 @@ package history_test import ( . "github.com/onsi/ginkgo" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/history" - "github.com/vulcanize/vulcanizedb/pkg/transformers" "math/big" ) @@ -15,24 +13,10 @@ var _ = Describe("Header validator", func() { headerRepository.SetMissingBlockNumbers([]int64{}) blockChain := fakes.NewMockBlockChain() blockChain.SetLastBlock(big.NewInt(3)) - validator := history.NewHeaderValidator(blockChain, headerRepository, 2, []transformers.Transformer{}) + validator := history.NewHeaderValidator(blockChain, headerRepository, 2) validator.ValidateHeaders() headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(3, []int64{1, 2, 3}) }) - - It("passes transformers for execution on new blocks", func() { - headerRepository := fakes.NewMockHeaderRepository() - headerRepository.SetMissingBlockNumbers([]int64{}) - blockChain := fakes.NewMockBlockChain() - blockChain.SetLastBlock(big.NewInt(3)) - transformer := fakes.NewMockTransformer() - validator := history.NewHeaderValidator(blockChain, headerRepository, 1, []transformers.Transformer{transformer}) - - validator.ValidateHeaders() - - headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(2, []int64{2, 3}) - transformer.AssertExecuteCalledWith(core.Header{BlockNumber: 3}, 0) - }) }) diff --git a/pkg/history/populate_headers.go b/pkg/history/populate_headers.go index 183bbf4d..c4e0612d 100644 --- a/pkg/history/populate_headers.go +++ b/pkg/history/populate_headers.go @@ -6,40 +6,33 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/transformers" ) -func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64, transformers []transformers.Transformer) (int, error) { +func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) { lastBlock := blockchain.LastBlock().Int64() blockRange := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock, blockchain.Node().ID) log.SetPrefix("") log.Printf("Backfilling %d blocks\n\n", len(blockRange)) - _, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange, transformers) + _, err := RetrieveAndUpdateHeaders(blockchain, headerRepository, blockRange) if err != nil { return 0, err } return len(blockRange), nil } -func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64, transformers []transformers.Transformer) (int, error) { +func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) { for _, blockNumber := range blockNumbers { header, err := chain.GetHeaderByNumber(blockNumber) if err != nil { return 0, err } - id, err := headerRepository.CreateOrUpdateHeader(header) + _, err = headerRepository.CreateOrUpdateHeader(header) if err != nil { if err == repositories.ErrValidHeaderExists { continue } return 0, err } - for _, transformer := range transformers { - err := transformer.Execute(header, id) - if err != nil { - return 0, err - } - } } return len(blockNumbers), nil } diff --git a/pkg/history/populate_headers_test.go b/pkg/history/populate_headers_test.go index 72a0cec9..009a620c 100644 --- a/pkg/history/populate_headers_test.go +++ b/pkg/history/populate_headers_test.go @@ -6,11 +6,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/history" - "github.com/vulcanize/vulcanizedb/pkg/transformers" ) var _ = Describe("Populating headers", func() { @@ -26,7 +23,7 @@ var _ = Describe("Populating headers", func() { blockChain.SetLastBlock(big.NewInt(2)) headerRepository.SetMissingBlockNumbers([]int64{2}) - headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1, []transformers.Transformer{}) + headersAdded, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1) Expect(err).NotTo(HaveOccurred()) Expect(headersAdded).To(Equal(1)) @@ -37,54 +34,9 @@ var _ = Describe("Populating headers", func() { blockChain.SetLastBlock(big.NewInt(2)) headerRepository.SetMissingBlockNumbers([]int64{2}) - _, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1, []transformers.Transformer{}) + _, err := history.PopulateMissingHeaders(blockChain, headerRepository, 1) Expect(err).NotTo(HaveOccurred()) headerRepository.AssertCreateOrUpdateHeaderCallCountAndPassedBlockNumbers(1, []int64{2}) }) - - It("executes passed transformers with created headers", func() { - blockNumber := int64(54321) - blockChain := fakes.NewMockBlockChain() - blockChain.SetLastBlock(big.NewInt(blockNumber)) - headerRepository.SetMissingBlockNumbers([]int64{blockNumber}) - headerID := int64(12345) - headerRepository.SetCreateOrUpdateHeaderReturnID(headerID) - transformer := fakes.NewMockTransformer() - - _, err := history.PopulateMissingHeaders(blockChain, headerRepository, blockNumber, []transformers.Transformer{transformer}) - - Expect(err).NotTo(HaveOccurred()) - transformer.AssertExecuteCalledWith(core.Header{BlockNumber: blockNumber}, headerID) - }) - - It("does not execute transformer if repository indicates header already exists", func() { - blockNumber := int64(54321) - blockChain := fakes.NewMockBlockChain() - blockChain.SetLastBlock(big.NewInt(blockNumber)) - headerRepository.SetMissingBlockNumbers([]int64{blockNumber}) - headerRepository.SetCreateOrUpdateHeaderReturnErr(repositories.ErrValidHeaderExists) - transformer := fakes.NewMockTransformer() - - _, err := history.PopulateMissingHeaders(blockChain, headerRepository, blockNumber, []transformers.Transformer{transformer}) - - Expect(err).NotTo(HaveOccurred()) - transformer.AssertExecuteNotCalled() - }) - - It("returns error if executing transformer fails", func() { - blockNumber := int64(54321) - blockChain := fakes.NewMockBlockChain() - blockChain.SetLastBlock(big.NewInt(blockNumber)) - headerRepository.SetMissingBlockNumbers([]int64{blockNumber}) - headerID := int64(12345) - headerRepository.SetCreateOrUpdateHeaderReturnID(headerID) - transformer := fakes.NewMockTransformer() - transformer.SetExecuteErr(fakes.FakeError) - - _, err := history.PopulateMissingHeaders(blockChain, headerRepository, blockNumber, []transformers.Transformer{transformer}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) }) diff --git a/pkg/transformers/flip_kick/config.go b/pkg/transformers/flip_kick/config.go index 9dbfbd70..8aef6634 100644 --- a/pkg/transformers/flip_kick/config.go +++ b/pkg/transformers/flip_kick/config.go @@ -14,16 +14,10 @@ package flip_kick -type TransformerConfig struct { - ContractAddress string - ContractAbi string - Topics []string - StartingBlockNumber int64 - EndingBlockNumber int64 -} +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var FlipKickConfig = TransformerConfig{ - ContractAddress: "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d", //this is a temporary address deployed locally +var FlipKickConfig = shared.TransformerConfig{ + ContractAddresses: "0x08cb6176addcca2e1d1ffe21bee464b72ee4cd8d", //this is a temporary address deployed locally ContractAbi: FlipperABI, Topics: []string{FlipKickSignature}, StartingBlockNumber: 0, diff --git a/pkg/transformers/flip_kick/transformer.go b/pkg/transformers/flip_kick/transformer.go index c5f40cb0..16ad9f3a 100644 --- a/pkg/transformers/flip_kick/transformer.go +++ b/pkg/transformers/flip_kick/transformer.go @@ -30,11 +30,11 @@ type FlipKickTransformer struct { Fetcher shared.LogFetcher Converter Converter Repository Repository - Config TransformerConfig + Config shared.TransformerConfig } type FlipKickTransformerInitializer struct { - Config TransformerConfig + Config shared.TransformerConfig } func (i FlipKickTransformerInitializer) NewFlipKickTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { @@ -50,7 +50,7 @@ func (i FlipKickTransformerInitializer) NewFlipKickTransformer(db *postgres.DB, return transformer } -func (fkt *FlipKickTransformer) SetConfig(config TransformerConfig) { +func (fkt *FlipKickTransformer) SetConfig(config shared.TransformerConfig) { fkt.Config = config } @@ -91,13 +91,13 @@ func (fkt FlipKickTransformer) Execute() error { log.Printf("Fetching event logs for %d headers \n", len(headers)) var resultingErrors []error for _, header := range headers { - ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddress, topics, header.BlockNumber) + ethLogs, err := fkt.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber) if err != nil { resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, FetcherError)) } for _, ethLog := range ethLogs { - entity, err := fkt.Converter.ToEntity(config.ContractAddress, config.ContractAbi, ethLog) + entity, err := fkt.Converter.ToEntity(config.ContractAddresses, config.ContractAbi, ethLog) if err != nil { resultingErrors = append(resultingErrors, newTransformerError(err, header.BlockNumber, LogToEntityError)) } diff --git a/pkg/transformers/flip_kick/transformer_test.go b/pkg/transformers/flip_kick/transformer_test.go index 4bb26475..293a1f31 100644 --- a/pkg/transformers/flip_kick/transformer_test.go +++ b/pkg/transformers/flip_kick/transformer_test.go @@ -25,6 +25,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" flip_kick_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/flip_kick" @@ -35,7 +36,7 @@ var _ = Describe("FlipKick Transformer", func() { var fetcher mocks.MockLogFetcher var converter flip_kick_mocks.MockFlipKickConverter var repository flip_kick_mocks.MockFlipKickRepository - var testConfig flip_kick.TransformerConfig + var testConfig shared.TransformerConfig var blockNumber int64 var headerId int64 var headers []core.Header @@ -52,8 +53,8 @@ var _ = Describe("FlipKick Transformer", func() { } startingBlockNumber := rand.Int63() - testConfig = flip_kick.TransformerConfig{ - ContractAddress: "0x12345", + testConfig = shared.TransformerConfig{ + ContractAddresses: "0x12345", ContractAbi: "test abi", Topics: []string{flip_kick.FlipKickSignature}, StartingBlockNumber: startingBlockNumber, @@ -82,7 +83,7 @@ var _ = Describe("FlipKick Transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddress)) + Expect(fetcher.FetchedContractAddress).To(Equal(testConfig.ContractAddresses)) Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber})) }) @@ -99,7 +100,7 @@ var _ = Describe("FlipKick Transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddress)) + Expect(converter.ConverterContract).To(Equal(testConfig.ContractAddresses)) Expect(converter.ConverterAbi).To(Equal(testConfig.ContractAbi)) Expect(converter.LogsToConvert).To(Equal(logs)) Expect(converter.EntitiesToConvert).To(Equal([]flip_kick.FlipKickEntity{test_data.FlipKickEntity})) diff --git a/pkg/transformers/frob/config.go b/pkg/transformers/frob/config.go index f10024a8..cd824f06 100644 --- a/pkg/transformers/frob/config.go +++ b/pkg/transformers/frob/config.go @@ -14,16 +14,10 @@ package frob -type TransformerConfig struct { - ContractAddress string - ContractAbi string - Topics []string - StartingBlockNumber int64 - EndingBlockNumber int64 -} +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var FrobConfig = TransformerConfig{ - ContractAddress: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally +var FrobConfig = shared.TransformerConfig{ + ContractAddresses: "0xff3f2400f1600f3f493a9a92704a29b96795af1a", //this is a temporary address deployed locally ContractAbi: FrobABI, Topics: []string{FrobEventSignature}, StartingBlockNumber: 0, diff --git a/pkg/transformers/frob/transformer.go b/pkg/transformers/frob/transformer.go index c1e957e5..f72d2b08 100644 --- a/pkg/transformers/frob/transformer.go +++ b/pkg/transformers/frob/transformer.go @@ -23,13 +23,14 @@ import ( ) type FrobTransformer struct { + Config shared.TransformerConfig Converter Converter Fetcher shared.LogFetcher Repository Repository } type FrobTransformerInitializer struct { - Config TransformerConfig + Config shared.TransformerConfig } func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { @@ -37,6 +38,7 @@ func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB fetcher := shared.NewFetcher(blockChain) repository := NewFrobRepository(db) return FrobTransformer{ + Config: initializer.Config, Converter: converter, Fetcher: fetcher, Repository: repository, @@ -44,18 +46,18 @@ func (initializer FrobTransformerInitializer) NewFrobTransformer(db *postgres.DB } func (transformer FrobTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(FrobConfig.StartingBlockNumber, FrobConfig.EndingBlockNumber) + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) if err != nil { return err } for _, header := range missingHeaders { topics := [][]common.Hash{{common.HexToHash(FrobEventSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddress, topics, header.BlockNumber) + matchingLogs, err := transformer.Fetcher.FetchLogs(FrobConfig.ContractAddresses, topics, header.BlockNumber) if err != nil { return err } for _, log := range matchingLogs { - entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddress, FrobConfig.ContractAbi, log) + entity, err := transformer.Converter.ToEntity(FrobConfig.ContractAddresses, FrobConfig.ContractAbi, log) if err != nil { return err } diff --git a/pkg/transformers/frob/transformer_test.go b/pkg/transformers/frob/transformer_test.go index d407de76..ca7dfe31 100644 --- a/pkg/transformers/frob/transformer_test.go +++ b/pkg/transformers/frob/transformer_test.go @@ -32,6 +32,7 @@ var _ = Describe("Frob transformer", func() { It("gets missing headers for block numbers specified in config", func() { repository := &frob_mocks.MockFrobRepository{} transformer := frob.FrobTransformer{ + Config: frob.FrobConfig, Fetcher: &mocks.MockLogFetcher{}, Converter: &frob_mocks.MockFrobConverter{}, Repository: repository, @@ -73,7 +74,7 @@ var _ = Describe("Frob transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddress)) + Expect(fetcher.FetchedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses)) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(frob.FrobEventSignature)}})) }) @@ -109,7 +110,7 @@ var _ = Describe("Frob transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddress)) + Expect(converter.PassedContractAddress).To(Equal(frob.FrobConfig.ContractAddresses)) Expect(converter.PassedContractABI).To(Equal(frob.FrobConfig.ContractAbi)) Expect(converter.PassedLog).To(Equal(test_data.EthFrobLog)) Expect(converter.PassedEntity).To(Equal(test_data.FrobEntity)) diff --git a/pkg/transformers/price_feeds/config.go b/pkg/transformers/price_feeds/config.go new file mode 100644 index 00000000..ddeabef3 --- /dev/null +++ b/pkg/transformers/price_feeds/config.go @@ -0,0 +1,37 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds + +var ( + PepAddress = "0x99041F808D598B782D5a3e498681C2452A31da08" + PipAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B" + RepAddress = "0xF5f94b7F9De14D43112e713835BCef2d55b76c1C" +) + +type IPriceFeedConfig struct { + ContractAddresses []string + StartingBlockNumber int64 + EndingBlockNumber int64 +} + +var PriceFeedConfig = IPriceFeedConfig{ + ContractAddresses: []string{ + PepAddress, + PipAddress, + RepAddress, + }, + StartingBlockNumber: 0, + EndingBlockNumber: 100, +} diff --git a/pkg/transformers/price_feeds/constants.go b/pkg/transformers/price_feeds/constants.go index dc7ecd86..846adf74 100644 --- a/pkg/transformers/price_feeds/constants.go +++ b/pkg/transformers/price_feeds/constants.go @@ -1,3 +1,17 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package price_feeds import ( @@ -6,13 +20,9 @@ import ( ) var ( - ErrMultipleLogs = errors.New("multiple matching logs found in block") ErrNoMatchingLog = errors.New("no matching log") Ether = big.NewFloat(1e18) - PeekMethodName = "peek" - PepLogTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527" - PipLogTopic0 = "0x1817835800000000000000000000000000000000000000000000000000000000" - PipMedianizerABI = `[{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"","type":"bytes32"}],"name":"poke","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"poke","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"compute","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"unset","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"indexes","outputs":[{"name":"","type":"bytes12"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"next","outputs":[{"name":"","type":"bytes12"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"read","outputs":[{"name":"","type":"bytes32"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"peek","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes12"}],"name":"values","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"min_","type":"uint96"}],"name":"setMin","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"void","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"},{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"}],"name":"unset","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"next_","type":"bytes12"}],"name":"setNext","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"min","outputs":[{"name":"","type":"uint96"}],"payable":false,"type":"function"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"}]` + LogValueTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527" + MedianizerABI = `[{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"","type":"bytes32"}],"name":"poke","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"poke","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"compute","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"unset","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"indexes","outputs":[{"name":"","type":"bytes12"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"next","outputs":[{"name":"","type":"bytes12"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"read","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"peek","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes12"}],"name":"values","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"min_","type":"uint96"}],"name":"setMin","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"void","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"},{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"}],"name":"unset","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"next_","type":"bytes12"}],"name":"setNext","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"min","outputs":[{"name":"","type":"uint96"}],"payable":false,"stateMutability":"view","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"val","type":"bytes32"}],"name":"LogValue","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"}]]` Ray = big.NewFloat(1e27) - RepLogTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527" ) diff --git a/pkg/transformers/price_feeds/converter.go b/pkg/transformers/price_feeds/converter.go new file mode 100644 index 00000000..443d9944 --- /dev/null +++ b/pkg/transformers/price_feeds/converter.go @@ -0,0 +1,32 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds + +import ( + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +type PriceFeedConverter struct{} + +func (converter PriceFeedConverter) ToModel(log types.Log, headerID int64) PriceFeedModel { + return PriceFeedModel{ + BlockNumber: log.BlockNumber, + HeaderID: headerID, + MedianizerAddress: log.Address.Bytes(), + UsdValue: Convert("wad", hexutil.Encode(log.Data), 15), + TransactionIndex: log.TxIndex, + } +} diff --git a/pkg/transformers/price_feeds/converter_test.go b/pkg/transformers/price_feeds/converter_test.go new file mode 100644 index 00000000..0c309477 --- /dev/null +++ b/pkg/transformers/price_feeds/converter_test.go @@ -0,0 +1,57 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" +) + +var _ = Describe("Price feed converter", func() { + It("converts a log to a price feed model", func() { + medianizerAddress := common.HexToAddress("0x99041f808d598b782d5a3e498681c2452a31da08") + blockNumber := uint64(6147230) + txIndex := uint(119) + // https://etherscan.io/tx/0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f + log := types.Log{ + Address: medianizerAddress, + Topics: []common.Hash{common.HexToHash(price_feeds.LogValueTopic0)}, + Data: common.FromHex("00000000000000000000000000000000000000000000001486f658319fb0c100"), + BlockNumber: blockNumber, + TxHash: common.HexToHash("0xa51a50a2adbfba4e2ab3d72dfd67a21c769f1bc8d2b180663a15500a56cde58f"), + TxIndex: txIndex, + BlockHash: common.HexToHash("0x27ecebbf69eefa3bb3cf65f472322a80ff4946653a50a2171dc605f49829467d"), + Index: 0, + Removed: false, + } + converter := price_feeds.PriceFeedConverter{} + headerID := int64(123) + + model := converter.ToModel(log, headerID) + + expectedModel := price_feeds.PriceFeedModel{ + BlockNumber: blockNumber, + HeaderID: headerID, + MedianizerAddress: medianizerAddress[:], + UsdValue: "378.6599388897", + TransactionIndex: txIndex, + } + Expect(model).To(Equal(expectedModel)) + }) +}) diff --git a/pkg/transformers/price_feeds/fetcher.go b/pkg/transformers/price_feeds/fetcher.go new file mode 100644 index 00000000..ecc79d4d --- /dev/null +++ b/pkg/transformers/price_feeds/fetcher.go @@ -0,0 +1,54 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds + +import ( + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "math/big" +) + +type IPriceFeedFetcher interface { + FetchLogValues(blockNumber int64) ([]types.Log, error) +} + +type PriceFeedFetcher struct { + blockChain core.BlockChain + contractAddresses []string +} + +func NewPriceFeedFetcher(blockChain core.BlockChain, contractAddresses []string) PriceFeedFetcher { + return PriceFeedFetcher{ + blockChain: blockChain, + contractAddresses: contractAddresses, + } +} + +func (fetcher PriceFeedFetcher) FetchLogValues(blockNumber int64) ([]types.Log, error) { + var addresses []common.Address + for _, addr := range fetcher.contractAddresses { + addresses = append(addresses, common.HexToAddress(addr)) + } + n := big.NewInt(blockNumber) + query := ethereum.FilterQuery{ + FromBlock: n, + ToBlock: n, + Addresses: addresses, + Topics: [][]common.Hash{{common.HexToHash(LogValueTopic0)}}, + } + return fetcher.blockChain.GetEthLogsWithCustomQuery(query) +} diff --git a/pkg/transformers/price_feeds/fetcher_test.go b/pkg/transformers/price_feeds/fetcher_test.go new file mode 100644 index 00000000..acb0acfb --- /dev/null +++ b/pkg/transformers/price_feeds/fetcher_test.go @@ -0,0 +1,64 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds_test + +import ( + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" +) + +var _ = Describe("Price fetcher", func() { + It("gets log value events from price feed medianizers", func() { + mockBlockChain := fakes.NewMockBlockChain() + mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}}) + contractAddresses := []string{"pep-contract-address", "pip-contract-address", "rep-contract-address"} + fetcher := price_feeds.NewPriceFeedFetcher(mockBlockChain, contractAddresses) + blockNumber := int64(100) + + _, err := fetcher.FetchLogValues(blockNumber) + + Expect(err).NotTo(HaveOccurred()) + var expectedAddresses []common.Address + for _, address := range contractAddresses { + expectedAddresses = append(expectedAddresses, common.HexToAddress(address)) + } + expectedQuery := ethereum.FilterQuery{ + FromBlock: big.NewInt(blockNumber), + ToBlock: big.NewInt(blockNumber), + Addresses: expectedAddresses, + Topics: [][]common.Hash{{common.HexToHash(price_feeds.LogValueTopic0)}}, + } + mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) + }) + + It("returns error if getting logs fails", func() { + mockBlockChain := fakes.NewMockBlockChain() + mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) + fetcher := price_feeds.NewPriceFeedFetcher(mockBlockChain, []string{"contract-address"}) + + _, err := fetcher.FetchLogValues(100) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/price_feeds/pep/fetcher.go b/pkg/transformers/price_feeds/pep/fetcher.go deleted file mode 100644 index 19b86738..00000000 --- a/pkg/transformers/price_feeds/pep/fetcher.go +++ /dev/null @@ -1,53 +0,0 @@ -package pep - -import ( - "math/big" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type IPepFetcher interface { - FetchPepValue(header core.Header) (string, error) -} - -type PepFetcher struct { - blockChain core.BlockChain - contractAddress string -} - -func NewPepFetcher(chain core.BlockChain, contractAddress string) PepFetcher { - return PepFetcher{ - blockChain: chain, - contractAddress: contractAddress, - } -} - -func (fetcher PepFetcher) FetchPepValue(header core.Header) (string, error) { - blockNumber := big.NewInt(header.BlockNumber) - query := ethereum.FilterQuery{ - FromBlock: blockNumber, - ToBlock: blockNumber, - Addresses: []common.Address{common.HexToAddress(fetcher.contractAddress)}, - Topics: [][]common.Hash{{common.HexToHash(price_feeds.PepLogTopic0)}}, - } - logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) - return fetcher.getLogValue(logs, err) -} - -func (fetcher PepFetcher) getLogValue(logs []types.Log, err error) (string, error) { - if err != nil { - return "", err - } - if len(logs) < 1 { - return "", price_feeds.ErrNoMatchingLog - } - if len(logs) > 1 { - return "", price_feeds.ErrMultipleLogs - } - return string(logs[0].Data), nil -} diff --git a/pkg/transformers/price_feeds/pep/fetcher_test.go b/pkg/transformers/price_feeds/pep/fetcher_test.go deleted file mode 100644 index 7342e9e8..00000000 --- a/pkg/transformers/price_feeds/pep/fetcher_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package pep_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" - "math/big" -) - -var _ = Describe("Pep fetcher", func() { - It("gets logs describing updated mkr/usd value", func() { - mockBlockChain := fakes.NewMockBlockChain() - mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}}) - contractAddress := "pep-contract-address" - fetcher := pep.NewPepFetcher(mockBlockChain, contractAddress) - blockNumber := int64(12345) - header := core.Header{ - BlockNumber: blockNumber, - Hash: "", - Raw: nil, - } - - _, err := fetcher.FetchPepValue(header) - - Expect(err).NotTo(HaveOccurred()) - expectedQuery := ethereum.FilterQuery{ - FromBlock: big.NewInt(blockNumber), - ToBlock: big.NewInt(blockNumber), - Addresses: []common.Address{common.HexToAddress(contractAddress)}, - Topics: [][]common.Hash{{common.HexToHash(price_feeds.PepLogTopic0)}}, - } - mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) - }) - - It("returns error if getting logs fails", func() { - mockBlockChain := fakes.NewMockBlockChain() - mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}}) - mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) - fetcher := pep.NewPepFetcher(mockBlockChain, "pep-contract-address") - - _, err := fetcher.FetchPepValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("returns no matching logs error if no logs returned", func() { - mockBlockChain := fakes.NewMockBlockChain() - fetcher := pep.NewPepFetcher(mockBlockChain, "pep-contract-address") - - _, err := fetcher.FetchPepValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog)) - }) - - It("returns error if more than one matching logs returned", func() { - mockBlockChain := fakes.NewMockBlockChain() - mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}, {}}) - fetcher := pep.NewPepFetcher(mockBlockChain, "pep-contract-address") - - _, err := fetcher.FetchPepValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(price_feeds.ErrMultipleLogs)) - }) -}) diff --git a/pkg/transformers/price_feeds/pep/pep_suite_test.go b/pkg/transformers/price_feeds/pep/pep_suite_test.go deleted file mode 100644 index 49dcb4cd..00000000 --- a/pkg/transformers/price_feeds/pep/pep_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package pep - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestPep(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Pep Suite") -} diff --git a/pkg/transformers/price_feeds/pep/repository.go b/pkg/transformers/price_feeds/pep/repository.go deleted file mode 100644 index 27832e0e..00000000 --- a/pkg/transformers/price_feeds/pep/repository.go +++ /dev/null @@ -1,25 +0,0 @@ -package pep - -import ( - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type IPepRepository interface { - CreatePep(pep price_feeds.PriceUpdate) error -} - -type PepRepository struct { - db *postgres.DB -} - -func NewPepRepository(db *postgres.DB) PepRepository { - return PepRepository{ - db: db, - } -} - -func (repository PepRepository) CreatePep(pep price_feeds.PriceUpdate) error { - _, err := repository.db.Exec(`INSERT INTO maker.peps (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, pep.BlockNumber, pep.HeaderID, pep.UsdValue) - return err -} diff --git a/pkg/transformers/price_feeds/pep/repository_test.go b/pkg/transformers/price_feeds/pep/repository_test.go deleted file mode 100644 index 0a092e27..00000000 --- a/pkg/transformers/price_feeds/pep/repository_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package pep_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("Pep repository", func() { - It("returns header if matching header does not exist", func() { - db := test_config.NewTestDB(core.Node{}) - repository := pep.NewPepRepository(db) - pepToAdd := price_feeds.PriceUpdate{ - BlockNumber: 0, - HeaderID: 0, - UsdValue: "123.456", - } - - err := repository.CreatePep(pepToAdd) - - Expect(err).To(HaveOccurred()) - }) - - It("creates a pep when matching header exists", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - repository := pep.NewPepRepository(db) - header := core.Header{BlockNumber: 12345} - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - pepToAdd := price_feeds.PriceUpdate{ - BlockNumber: header.BlockNumber, - HeaderID: headerID, - UsdValue: "123.456", - } - - err = repository.CreatePep(pepToAdd) - - Expect(err).NotTo(HaveOccurred()) - var dbPep price_feeds.PriceUpdate - err = db.Get(&dbPep, `SELECT block_number, header_id, usd_value FROM maker.peps WHERE header_id = $1`, pepToAdd.HeaderID) - Expect(err).NotTo(HaveOccurred()) - Expect(dbPep).To(Equal(pepToAdd)) - }) -}) diff --git a/pkg/transformers/price_feeds/pep/transformer.go b/pkg/transformers/price_feeds/pep/transformer.go deleted file mode 100644 index 8f83f4de..00000000 --- a/pkg/transformers/price_feeds/pep/transformer.go +++ /dev/null @@ -1,43 +0,0 @@ -package pep - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type PepTransformer struct { - fetcher IPepFetcher - repository IPepRepository -} - -func NewPepTransformer(chain core.BlockChain, db *postgres.DB, contractAddress string) PepTransformer { - fetcher := NewPepFetcher(chain, contractAddress) - repository := NewPepRepository(db) - return PepTransformer{ - fetcher: fetcher, - repository: repository, - } -} - -func (transformer PepTransformer) Execute(header core.Header, headerID int64) error { - logValue, err := transformer.fetcher.FetchPepValue(header) - if err != nil { - if err == price_feeds.ErrNoMatchingLog { - return nil - } - return err - } - pep := getPep(logValue, header, headerID) - return transformer.repository.CreatePep(pep) -} - -func getPep(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate { - valueInUSD := price_feeds.Convert("wad", logValue, 15) - pep := price_feeds.PriceUpdate{ - BlockNumber: header.BlockNumber, - HeaderID: headerID, - UsdValue: valueInUSD, - } - return pep -} diff --git a/pkg/transformers/price_feeds/pep/transformer_test.go b/pkg/transformers/price_feeds/pep/transformer_test.go deleted file mode 100644 index 746aa0ca..00000000 --- a/pkg/transformers/price_feeds/pep/transformer_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package pep_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("Pep transformer", func() { - It("returns nil if no logs found", func() { - chain := fakes.NewMockBlockChain() - db := test_config.NewTestDB(core.Node{}) - transformer := pep.NewPepTransformer(chain, db, "pep-contract-address") - - err := transformer.Execute(core.Header{}, 123) - - Expect(err).NotTo(HaveOccurred()) - }) - - It("creates pep row for found log", func() { - chain := fakes.NewMockBlockChain() - chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}}) - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - header := core.Header{BlockNumber: 12345} - headerID, err := headerRepository.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - transformer := pep.NewPepTransformer(chain, db, "pep-contract-address") - - err = transformer.Execute(header, headerID) - - Expect(err).NotTo(HaveOccurred()) - var dbPep price_feeds.PriceUpdate - err = db.Get(&dbPep, `SELECT block_number, header_id, usd_value FROM maker.peps WHERE header_id = $1`, headerID) - Expect(err).NotTo(HaveOccurred()) - Expect(dbPep.BlockNumber).To(Equal(header.BlockNumber)) - }) -}) diff --git a/pkg/transformers/price_feeds/pip/fetcher.go b/pkg/transformers/price_feeds/pip/fetcher.go deleted file mode 100644 index 08c8a321..00000000 --- a/pkg/transformers/price_feeds/pip/fetcher.go +++ /dev/null @@ -1,77 +0,0 @@ -package pip - -import ( - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "math/big" -) - -type IPipFetcher interface { - FetchPipValue(header core.Header) (string, error) -} - -type PipFetcher struct { - blockChain core.BlockChain - contractAddress string -} - -func NewPipFetcher(chain core.BlockChain, contractAddress string) PipFetcher { - return PipFetcher{ - blockChain: chain, - contractAddress: contractAddress, - } -} - -func (fetcher PipFetcher) FetchPipValue(header core.Header) (string, error) { - blockNumber := big.NewInt(header.BlockNumber) - query := ethereum.FilterQuery{ - FromBlock: blockNumber, - ToBlock: blockNumber, - Addresses: []common.Address{common.HexToAddress(fetcher.contractAddress)}, - Topics: [][]common.Hash{{common.HexToHash(price_feeds.PipLogTopic0)}}, - } - logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) - if err != nil { - return "", err - } - if len(logs) > 0 { - return fetcher.getLogValue(logs, err) - } - return "", price_feeds.ErrNoMatchingLog -} - -func (fetcher PipFetcher) getLogValue(logs []types.Log, err error) (string, error) { - var ( - ret0 = new([32]byte) - ret1 = new(bool) - ) - var r = &[]interface{}{ - ret0, - ret1, - } - err = fetcher.blockChain.FetchContractData(price_feeds.PipMedianizerABI, fetcher.contractAddress, price_feeds.PeekMethodName, nil, r, int64(logs[0].BlockNumber)) - if err != nil { - return "", err - } - result := newResult(*ret0, *ret1) - return result.Value.String(), nil -} - -type Value [32]byte - -type Peek struct { - Value - OK bool -} - -func (value Value) String() string { - bi := big.NewInt(0).SetBytes(value[:]) - return bi.String() -} - -func newResult(value [32]byte, ok bool) *Peek { - return &Peek{Value: value, OK: ok} -} diff --git a/pkg/transformers/price_feeds/pip/fetcher_test.go b/pkg/transformers/price_feeds/pip/fetcher_test.go deleted file mode 100644 index e9cdc053..00000000 --- a/pkg/transformers/price_feeds/pip/fetcher_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package pip_test - -import ( - "github.com/ethereum/go-ethereum/core/types" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" -) - -var _ = Describe("Pip fetcher", func() { - It("returns error if fetching logs fails", func() { - chain := fakes.NewMockBlockChain() - chain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) - fetcher := pip.NewPipFetcher(chain, "pip-contract-address") - - _, err := fetcher.FetchPipValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("returns no matching logs error if no logs returned", func() { - chain := fakes.NewMockBlockChain() - fetcher := pip.NewPipFetcher(chain, "pip-contract-address") - - _, err := fetcher.FetchPipValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog)) - }) - - Describe("when matching log found", func() { - It("calls contract to peek current eth/usd value", func() { - blockNumber := uint64(12345) - chain := fakes.NewMockBlockChain() - chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{BlockNumber: blockNumber}}) - contractAddress := "pip-contract-address" - fetcher := pip.NewPipFetcher(chain, contractAddress) - - _, err := fetcher.FetchPipValue(core.Header{}) - - Expect(err).NotTo(HaveOccurred()) - chain.AssertFetchContractDataCalledWith(price_feeds.PipMedianizerABI, contractAddress, price_feeds.PeekMethodName, nil, &[]interface{}{[32]byte{}, false}, int64(blockNumber)) - }) - - It("returns error if contract call fails", func() { - chain := fakes.NewMockBlockChain() - chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{BlockNumber: uint64(12345)}}) - chain.SetFetchContractDataErr(fakes.FakeError) - fetcher := pip.NewPipFetcher(chain, "pip-contract-address") - - _, err := fetcher.FetchPipValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - }) -}) diff --git a/pkg/transformers/price_feeds/pip/pip_suite_test.go b/pkg/transformers/price_feeds/pip/pip_suite_test.go deleted file mode 100644 index 30668aa9..00000000 --- a/pkg/transformers/price_feeds/pip/pip_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package pip - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestPip(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Pip Suite") -} diff --git a/pkg/transformers/price_feeds/pip/repository.go b/pkg/transformers/price_feeds/pip/repository.go deleted file mode 100644 index b6f3edc5..00000000 --- a/pkg/transformers/price_feeds/pip/repository.go +++ /dev/null @@ -1,25 +0,0 @@ -package pip - -import ( - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type IPipRepository interface { - CreatePip(pip price_feeds.PriceUpdate) error -} - -type PipRepository struct { - db *postgres.DB -} - -func NewPipRepository(db *postgres.DB) PipRepository { - return PipRepository{ - db: db, - } -} - -func (repository PipRepository) CreatePip(pip price_feeds.PriceUpdate) error { - _, err := repository.db.Exec(`INSERT INTO maker.pips (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, pip.BlockNumber, pip.HeaderID, pip.UsdValue) - return err -} diff --git a/pkg/transformers/price_feeds/pip/repository_test.go b/pkg/transformers/price_feeds/pip/repository_test.go deleted file mode 100644 index c47c0e36..00000000 --- a/pkg/transformers/price_feeds/pip/repository_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package pip_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("Pip repository", func() { - It("does not create a pip if no matching header", func() { - db := test_config.NewTestDB(core.Node{}) - repository := pip.NewPipRepository(db) - priceUpdate := price_feeds.PriceUpdate{ - BlockNumber: 0, - HeaderID: 0, - UsdValue: "123", - } - - err := repository.CreatePip(priceUpdate) - - Expect(err).To(HaveOccurred()) - }) - - It("creates a pip when header exists", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - repository := pip.NewPipRepository(db) - headerRepository := repositories.NewHeaderRepository(db) - header := core.Header{BlockNumber: 12345} - headerID, err := headerRepository.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - priceUpdate := price_feeds.PriceUpdate{ - BlockNumber: header.BlockNumber, - HeaderID: headerID, - UsdValue: "777.777", - } - - err = repository.CreatePip(priceUpdate) - - Expect(err).NotTo(HaveOccurred()) - var dbPip price_feeds.PriceUpdate - err = db.Get(&dbPip, `SELECT block_number, header_id, usd_value FROM maker.pips WHERE block_number = $1`, header.BlockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(dbPip).To(Equal(priceUpdate)) - }) -}) diff --git a/pkg/transformers/price_feeds/pip/transformer.go b/pkg/transformers/price_feeds/pip/transformer.go deleted file mode 100644 index 03845e8d..00000000 --- a/pkg/transformers/price_feeds/pip/transformer.go +++ /dev/null @@ -1,43 +0,0 @@ -package pip - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type PipTransformer struct { - fetcher IPipFetcher - repository IPipRepository -} - -func NewPipTransformer(chain core.BlockChain, db *postgres.DB, contractAddress string) PipTransformer { - fetcher := NewPipFetcher(chain, contractAddress) - repository := NewPipRepository(db) - return PipTransformer{ - fetcher: fetcher, - repository: repository, - } -} - -func (transformer PipTransformer) Execute(header core.Header, headerID int64) error { - value, err := transformer.fetcher.FetchPipValue(header) - if err != nil { - if err == price_feeds.ErrNoMatchingLog { - return nil - } - return err - } - pip := getPip(value, header, headerID) - return transformer.repository.CreatePip(pip) -} - -func getPip(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate { - valueInUSD := price_feeds.Convert("wad", logValue, 15) - pep := price_feeds.PriceUpdate{ - BlockNumber: header.BlockNumber, - HeaderID: headerID, - UsdValue: valueInUSD, - } - return pep -} diff --git a/pkg/transformers/price_feeds/pip/transformer_test.go b/pkg/transformers/price_feeds/pip/transformer_test.go deleted file mode 100644 index be6ef755..00000000 --- a/pkg/transformers/price_feeds/pip/transformer_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package pip_test - -import ( - "github.com/ethereum/go-ethereum/core/types" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("Pip transformer", func() { - It("returns nil if no logs found", func() { - chain := fakes.NewMockBlockChain() - db := test_config.NewTestDB(core.Node{}) - transformer := pip.NewPipTransformer(chain, db, "pip-contract-address") - - err := transformer.Execute(core.Header{}, 123) - - Expect(err).NotTo(HaveOccurred()) - }) - - It("creates pip row for found log", func() { - chain := fakes.NewMockBlockChain() - chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}}) - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - header := core.Header{BlockNumber: 12345} - headerID, err := headerRepository.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - transformer := pip.NewPipTransformer(chain, db, "pip-contract-address") - - err = transformer.Execute(header, headerID) - - Expect(err).NotTo(HaveOccurred()) - var dbPip price_feeds.PriceUpdate - err = db.Get(&dbPip, `SELECT block_number, header_id, usd_value FROM maker.pips WHERE header_id = $1`, headerID) - Expect(err).NotTo(HaveOccurred()) - Expect(dbPip.BlockNumber).To(Equal(header.BlockNumber)) - }) -}) diff --git a/pkg/transformers/price_feeds/price_feeds_suite_test.go b/pkg/transformers/price_feeds/price_feeds_suite_test.go new file mode 100644 index 00000000..e337dff0 --- /dev/null +++ b/pkg/transformers/price_feeds/price_feeds_suite_test.go @@ -0,0 +1,27 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPriceFeeds(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "PriceFeeds Suite") +} diff --git a/pkg/transformers/price_feeds/price_update.go b/pkg/transformers/price_feeds/price_update.go index cd70066a..d052cda5 100644 --- a/pkg/transformers/price_feeds/price_update.go +++ b/pkg/transformers/price_feeds/price_update.go @@ -1,11 +1,34 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package price_feeds -import "math/big" +import ( + "github.com/ethereum/go-ethereum/common" + "math/big" +) -type PriceUpdate struct { - BlockNumber int64 `db:"block_number"` - HeaderID int64 `db:"header_id"` - UsdValue string `db:"usd_value"` +type LogValueEntity struct { + Val common.Address +} + +type PriceFeedModel struct { + BlockNumber uint64 `db:"block_number"` + HeaderID int64 `db:"header_id"` + MedianizerAddress []byte `db:"medianizer_address"` + UsdValue string `db:"usd_value"` + TransactionIndex uint `db:"tx_idx"` } func Convert(conversion string, value string, prec int) string { diff --git a/pkg/transformers/price_feeds/rep/fetcher.go b/pkg/transformers/price_feeds/rep/fetcher.go deleted file mode 100644 index f4f0ce68..00000000 --- a/pkg/transformers/price_feeds/rep/fetcher.go +++ /dev/null @@ -1,51 +0,0 @@ -package rep - -import ( - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "math/big" -) - -type IRepFetcher interface { - FetchRepValue(header core.Header) (string, error) -} - -type RepFetcher struct { - chain core.BlockChain - contractAddress string -} - -func NewRepFetcher(chain core.BlockChain, contractAddress string) RepFetcher { - return RepFetcher{ - chain: chain, - contractAddress: contractAddress, - } -} - -func (fetcher RepFetcher) FetchRepValue(header core.Header) (string, error) { - blockNumber := big.NewInt(header.BlockNumber) - query := ethereum.FilterQuery{ - FromBlock: blockNumber, - ToBlock: blockNumber, - Addresses: []common.Address{common.HexToAddress(fetcher.contractAddress)}, - Topics: [][]common.Hash{{common.HexToHash(price_feeds.RepLogTopic0)}}, - } - logs, err := fetcher.chain.GetEthLogsWithCustomQuery(query) - return fetcher.getLogValue(logs, err) -} - -func (fetcher RepFetcher) getLogValue(logs []types.Log, err error) (string, error) { - if err != nil { - return "", err - } - if len(logs) < 1 { - return "", price_feeds.ErrNoMatchingLog - } - if len(logs) > 1 { - return "", price_feeds.ErrMultipleLogs - } - return string(logs[0].Data), nil -} diff --git a/pkg/transformers/price_feeds/rep/fetcher_test.go b/pkg/transformers/price_feeds/rep/fetcher_test.go deleted file mode 100644 index 5b9a3796..00000000 --- a/pkg/transformers/price_feeds/rep/fetcher_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package rep_test - -import ( - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep" - "math/big" -) - -var _ = Describe("Rep fetcher", func() { - It("gets logs describing updated rep/usd value", func() { - mockBlockChain := fakes.NewMockBlockChain() - mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}}) - contractAddress := "rep-contract-address" - fetcher := rep.NewRepFetcher(mockBlockChain, contractAddress) - blockNumber := int64(100) - header := core.Header{ - BlockNumber: blockNumber, - Hash: "", - Raw: nil, - } - - _, err := fetcher.FetchRepValue(header) - - Expect(err).NotTo(HaveOccurred()) - expectedQuery := ethereum.FilterQuery{ - FromBlock: big.NewInt(blockNumber), - ToBlock: big.NewInt(blockNumber), - Addresses: []common.Address{common.HexToAddress(contractAddress)}, - Topics: [][]common.Hash{{common.HexToHash(price_feeds.RepLogTopic0)}}, - } - mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) - }) - - It("returns error if getting logs fails", func() { - mockBlockChain := fakes.NewMockBlockChain() - mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) - fetcher := rep.NewRepFetcher(mockBlockChain, "rep-contract-address") - - _, err := fetcher.FetchRepValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("returns no matching logs error if no logs returned", func() { - mockBlockChain := fakes.NewMockBlockChain() - fetcher := rep.NewRepFetcher(mockBlockChain, "rep-contract-address") - - _, err := fetcher.FetchRepValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog)) - }) - - It("returns error if more than one matching logs returned", func() { - mockBlockChain := fakes.NewMockBlockChain() - mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}, {}}) - fetcher := rep.NewRepFetcher(mockBlockChain, "rep-contract-address") - - _, err := fetcher.FetchRepValue(core.Header{}) - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(price_feeds.ErrMultipleLogs)) - }) -}) diff --git a/pkg/transformers/price_feeds/rep/rep_suite_test.go b/pkg/transformers/price_feeds/rep/rep_suite_test.go deleted file mode 100644 index a4860d32..00000000 --- a/pkg/transformers/price_feeds/rep/rep_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package rep_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestRep(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Rep Suite") -} diff --git a/pkg/transformers/price_feeds/rep/repository.go b/pkg/transformers/price_feeds/rep/repository.go deleted file mode 100644 index eaed6597..00000000 --- a/pkg/transformers/price_feeds/rep/repository.go +++ /dev/null @@ -1,25 +0,0 @@ -package rep - -import ( - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type IRepRepository interface { - CreateRep(rep price_feeds.PriceUpdate) error -} - -type RepRepository struct { - db *postgres.DB -} - -func NewRepRepository(db *postgres.DB) RepRepository { - return RepRepository{ - db: db, - } -} - -func (repository RepRepository) CreateRep(rep price_feeds.PriceUpdate) error { - _, err := repository.db.Exec(`INSERT INTO maker.reps (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, rep.BlockNumber, rep.HeaderID, rep.UsdValue) - return err -} diff --git a/pkg/transformers/price_feeds/rep/repository_test.go b/pkg/transformers/price_feeds/rep/repository_test.go deleted file mode 100644 index 0ed535d5..00000000 --- a/pkg/transformers/price_feeds/rep/repository_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package rep_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("Rep repository", func() { - It("returns header if matching header does not exist", func() { - db := test_config.NewTestDB(core.Node{}) - repository := rep.NewRepRepository(db) - pepToAdd := price_feeds.PriceUpdate{ - BlockNumber: 0, - HeaderID: 0, - UsdValue: "123.456", - } - - err := repository.CreateRep(pepToAdd) - - Expect(err).To(HaveOccurred()) - }) - - It("creates a rep when matching header exists", func() { - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - repository := rep.NewRepRepository(db) - header := core.Header{BlockNumber: 12345} - headerRepository := repositories.NewHeaderRepository(db) - headerID, err := headerRepository.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - pepToAdd := price_feeds.PriceUpdate{ - BlockNumber: header.BlockNumber, - HeaderID: headerID, - UsdValue: "123.456", - } - - err = repository.CreateRep(pepToAdd) - - Expect(err).NotTo(HaveOccurred()) - var dbRep price_feeds.PriceUpdate - err = db.Get(&dbRep, `SELECT block_number, header_id, usd_value FROM maker.reps WHERE header_id = $1`, pepToAdd.HeaderID) - Expect(err).NotTo(HaveOccurred()) - Expect(dbRep).To(Equal(pepToAdd)) - }) -}) diff --git a/pkg/transformers/price_feeds/rep/transformer.go b/pkg/transformers/price_feeds/rep/transformer.go deleted file mode 100644 index fe692c5d..00000000 --- a/pkg/transformers/price_feeds/rep/transformer.go +++ /dev/null @@ -1,43 +0,0 @@ -package rep - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" -) - -type RepTransformer struct { - fetcher IRepFetcher - repository IRepRepository -} - -func NewRepTransformer(chain core.BlockChain, db *postgres.DB, contractAddress string) RepTransformer { - fetcher := NewRepFetcher(chain, contractAddress) - repository := NewRepRepository(db) - return RepTransformer{ - fetcher: fetcher, - repository: repository, - } -} - -func (transformer RepTransformer) Execute(header core.Header, headerID int64) error { - logValue, err := transformer.fetcher.FetchRepValue(header) - if err != nil { - if err == price_feeds.ErrNoMatchingLog { - return nil - } - return err - } - rep := getRep(logValue, header, headerID) - return transformer.repository.CreateRep(rep) -} - -func getRep(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate { - valueInUSD := price_feeds.Convert("wad", logValue, 15) - rep := price_feeds.PriceUpdate{ - BlockNumber: header.BlockNumber, - HeaderID: headerID, - UsdValue: valueInUSD, - } - return rep -} diff --git a/pkg/transformers/price_feeds/rep/transformer_test.go b/pkg/transformers/price_feeds/rep/transformer_test.go deleted file mode 100644 index 5bcce1e3..00000000 --- a/pkg/transformers/price_feeds/rep/transformer_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package rep_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" - "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/rep" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("Rep transformer", func() { - It("returns nil if no logs found", func() { - chain := fakes.NewMockBlockChain() - db := test_config.NewTestDB(core.Node{}) - transformer := rep.NewRepTransformer(chain, db, "rep-contract-address") - - err := transformer.Execute(core.Header{}, 123) - - Expect(err).NotTo(HaveOccurred()) - }) - - It("creates rep row for found log", func() { - chain := fakes.NewMockBlockChain() - chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}}) - db := test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - header := core.Header{BlockNumber: 12345} - headerID, err := headerRepository.CreateOrUpdateHeader(header) - Expect(err).NotTo(HaveOccurred()) - transformer := rep.NewRepTransformer(chain, db, "rep-contract-address") - - err = transformer.Execute(header, headerID) - - Expect(err).NotTo(HaveOccurred()) - var dbRep price_feeds.PriceUpdate - err = db.Get(&dbRep, `SELECT block_number, header_id, usd_value FROM maker.reps WHERE header_id = $1`, headerID) - Expect(err).NotTo(HaveOccurred()) - Expect(dbRep.BlockNumber).To(Equal(header.BlockNumber)) - }) -}) diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go new file mode 100644 index 00000000..77ea16e5 --- /dev/null +++ b/pkg/transformers/price_feeds/repository.go @@ -0,0 +1,56 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type IPriceFeedRepository interface { + Create(model PriceFeedModel) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + +type PriceFeedRepository struct { + db *postgres.DB +} + +func NewPriceFeedRepository(db *postgres.DB) PriceFeedRepository { + return PriceFeedRepository{db: db} +} + +func (repository PriceFeedRepository) Create(model PriceFeedModel) error { + _, err := repository.db.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, tx_idx) + VALUES ($1, $2, $3, $4::NUMERIC, $5)`, model.BlockNumber, model.HeaderID, model.MedianizerAddress, model.UsdValue, model.TransactionIndex) + return err +} + +func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + var result []core.Header + err := repository.db.Select( + &result, + `SELECT headers.id, headers.block_number FROM headers + LEFT JOIN maker.price_feeds on headers.id = header_id + WHERE header_id ISNULL + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlockNumber, + endingBlockNumber, + repository.db.Node.ID, + ) + return result, err +} diff --git a/pkg/transformers/price_feeds/repository_test.go b/pkg/transformers/price_feeds/repository_test.go new file mode 100644 index 00000000..dffa4b17 --- /dev/null +++ b/pkg/transformers/price_feeds/repository_test.go @@ -0,0 +1,147 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Price feeds repository", func() { + Describe("Create", func() { + It("persists a price feed update", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + blockNumber := uint64(12345) + header := core.Header{BlockNumber: int64(blockNumber)} + headerID, err := headerRepository.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + priceFeedUpdate := price_feeds.PriceFeedModel{ + BlockNumber: blockNumber, + HeaderID: headerID, + MedianizerAddress: []byte{1, 2, 3, 4, 5}, + UsdValue: "123.45", + TransactionIndex: 1, + } + priceFeedRepository := price_feeds.NewPriceFeedRepository(db) + + err = priceFeedRepository.Create(priceFeedUpdate) + + Expect(err).NotTo(HaveOccurred()) + var dbPriceFeedUpdate price_feeds.PriceFeedModel + err = db.Get(&dbPriceFeedUpdate, `SELECT block_number, header_id, medianizer_address, usd_value, tx_idx FROM maker.price_feeds WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbPriceFeedUpdate).To(Equal(priceFeedUpdate)) + }) + + It("does not duplicate price feed updates", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + blockNumber := uint64(12345) + header := core.Header{BlockNumber: int64(blockNumber)} + headerID, err := headerRepository.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + priceFeedUpdate := price_feeds.PriceFeedModel{ + BlockNumber: blockNumber, + HeaderID: headerID, + MedianizerAddress: []byte{1, 2, 3, 4, 5}, + UsdValue: "123.45", + TransactionIndex: 1, + } + priceFeedRepository := price_feeds.NewPriceFeedRepository(db) + err = priceFeedRepository.Create(priceFeedUpdate) + Expect(err).NotTo(HaveOccurred()) + + err = priceFeedRepository.Create(priceFeedUpdate) + + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers with no associated price feed event", func() { + node := core.Node{} + db := test_config.NewTestDB(node) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + startingBlockNumber := int64(1) + priceFeedBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers := []int64{startingBlockNumber, priceFeedBlockNumber, endingBlockNumber, endingBlockNumber + 1} + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + priceFeedRepository := price_feeds.NewPriceFeedRepository(db) + priceFeedUpdate := price_feeds.PriceFeedModel{ + BlockNumber: uint64(blockNumbers[1]), + HeaderID: headerIDs[1], + UsdValue: "123.45", + } + err := priceFeedRepository.Create(priceFeedUpdate) + Expect(err).NotTo(HaveOccurred()) + + headers, err := priceFeedRepository.MissingHeaders(startingBlockNumber, endingBlockNumber) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(2)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + nodeOne := core.Node{} + db := test_config.NewTestDB(nodeOne) + test_config.CleanTestDB(db) + blockNumbers := []int64{1, 2, 3} + headerRepository := repositories.NewHeaderRepository(db) + nodeTwo := core.Node{ID: "second"} + dbTwo := test_config.NewTestDB(nodeTwo) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + var headerIDs []int64 + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIDs = append(headerIDs, headerID) + _, err = headerRepositoryTwo.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + priceFeedRepository := price_feeds.NewPriceFeedRepository(db) + priceFeedRepositoryTwo := price_feeds.NewPriceFeedRepository(dbTwo) + err := priceFeedRepository.Create(price_feeds.PriceFeedModel{ + HeaderID: headerIDs[0], + UsdValue: "123.45", + }) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := priceFeedRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) + + nodeTwoMissingHeaders, err := priceFeedRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) + }) + }) +}) diff --git a/pkg/transformers/price_feeds/transformer.go b/pkg/transformers/price_feeds/transformer.go new file mode 100644 index 00000000..5261f0f1 --- /dev/null +++ b/pkg/transformers/price_feeds/transformer.go @@ -0,0 +1,65 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type PriceFeedTransformerInitializer struct { + Config IPriceFeedConfig +} + +func (initializer PriceFeedTransformerInitializer) NewPriceFeedTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := PriceFeedConverter{} + fetcher := NewPriceFeedFetcher(blockChain, initializer.Config.ContractAddresses) + repository := NewPriceFeedRepository(db) + return PriceFeedTransformer{ + Config: initializer.Config, + converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type PriceFeedTransformer struct { + Config IPriceFeedConfig + converter PriceFeedConverter + Fetcher IPriceFeedFetcher + Repository IPriceFeedRepository +} + +func (transformer PriceFeedTransformer) Execute() error { + headers, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + for _, header := range headers { + logs, err := transformer.Fetcher.FetchLogValues(header.BlockNumber) + if err != nil { + return err + } + for _, log := range logs { + model := transformer.converter.ToModel(log, header.Id) + err = transformer.Repository.Create(model) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/transformers/price_feeds/transformer_test.go b/pkg/transformers/price_feeds/transformer_test.go new file mode 100644 index 00000000..675694c2 --- /dev/null +++ b/pkg/transformers/price_feeds/transformer_test.go @@ -0,0 +1,150 @@ +// Copyright © 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package price_feeds_test + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + price_feeds2 "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/price_feeds" + "math/big" +) + +var _ = Describe("Price feed transformer", func() { + It("gets missing headers for price feeds", func() { + mockRepository := &price_feeds2.MockPriceFeedRepository{} + transformer := price_feeds.PriceFeedTransformer{ + Config: price_feeds.PriceFeedConfig, + Fetcher: &price_feeds2.MockPriceFeedFetcher{}, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockRepository.AssertMissingHeadersCalledwith(price_feeds.PriceFeedConfig.StartingBlockNumber, price_feeds.PriceFeedConfig.EndingBlockNumber) + }) + + It("returns error is missing headers call returns err", func() { + mockRepository := &price_feeds2.MockPriceFeedRepository{} + mockRepository.SetMissingHeadersErr(fakes.FakeError) + transformer := price_feeds.PriceFeedTransformer{ + Fetcher: &price_feeds2.MockPriceFeedFetcher{}, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + mockRepository := &price_feeds2.MockPriceFeedRepository{} + blockNumberOne := int64(1) + blockNumberTwo := int64(2) + mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumberOne}, {BlockNumber: blockNumberTwo}}) + mockFetcher := &price_feeds2.MockPriceFeedFetcher{} + transformer := price_feeds.PriceFeedTransformer{ + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + mockFetcher.AssertFetchLogValuesCalledWith([]int64{blockNumberOne, blockNumberTwo}) + }) + + It("returns err if fetcher returns err", func() { + mockRepository := &price_feeds2.MockPriceFeedRepository{} + mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + mockFetcher := &price_feeds2.MockPriceFeedFetcher{} + mockFetcher.SetReturnErr(fakes.FakeError) + transformer := price_feeds.PriceFeedTransformer{ + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists model converted from log", func() { + mockRepository := &price_feeds2.MockPriceFeedRepository{} + headerID := int64(11111) + mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: headerID}}) + mockFetcher := &price_feeds2.MockPriceFeedFetcher{} + blockNumber := uint64(22222) + txIndex := uint(33333) + usdValue := int64(44444) + etherMultiplier, _ := price_feeds.Ether.Int64() + rawUsdValue := big.NewInt(0) + rawUsdValue = rawUsdValue.Mul(big.NewInt(usdValue), big.NewInt(etherMultiplier)) + address := common.BytesToAddress([]byte{1, 2, 3, 4, 5}) + fakeLog := types.Log{ + Address: address, + Topics: nil, + Data: rawUsdValue.Bytes(), + BlockNumber: blockNumber, + TxHash: common.Hash{}, + TxIndex: txIndex, + BlockHash: common.Hash{}, + Index: 0, + Removed: false, + } + mockFetcher.SetReturnLogs([]types.Log{fakeLog}) + transformer := price_feeds.PriceFeedTransformer{ + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + expectedModel := price_feeds.PriceFeedModel{ + BlockNumber: blockNumber, + HeaderID: headerID, + MedianizerAddress: address.Bytes(), + UsdValue: fmt.Sprintf("%d", usdValue), + TransactionIndex: txIndex, + } + mockRepository.AssertCreateCalledWith(expectedModel) + }) + + It("returns error if creating price feed update returns error", func() { + mockRepository := &price_feeds2.MockPriceFeedRepository{} + mockRepository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + mockRepository.SetCreateErr(fakes.FakeError) + mockFetcher := &price_feeds2.MockPriceFeedFetcher{} + mockFetcher.SetReturnLogs([]types.Log{{}}) + transformer := price_feeds.PriceFeedTransformer{ + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index e5b32743..56f8fc40 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -27,6 +27,14 @@ type Transformer interface { type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer +type TransformerConfig struct { + ContractAddresses string + ContractAbi string + Topics []string + StartingBlockNumber int64 + EndingBlockNumber int64 +} + func HexToInt64(byteString string) int64 { value := common.HexToHash(byteString) return value.Big().Int64() diff --git a/pkg/transformers/test_data/mocks/price_feeds/fetcher.go b/pkg/transformers/test_data/mocks/price_feeds/fetcher.go new file mode 100644 index 00000000..95ca63d4 --- /dev/null +++ b/pkg/transformers/test_data/mocks/price_feeds/fetcher.go @@ -0,0 +1,29 @@ +package price_feeds + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/gomega" +) + +type MockPriceFeedFetcher struct { + passedBlockNumbers []int64 + returnErr error + returnLogs []types.Log +} + +func (fetcher *MockPriceFeedFetcher) SetReturnErr(err error) { + fetcher.returnErr = err +} + +func (fetcher *MockPriceFeedFetcher) SetReturnLogs(logs []types.Log) { + fetcher.returnLogs = logs +} + +func (fetcher *MockPriceFeedFetcher) FetchLogValues(blockNumber int64) ([]types.Log, error) { + fetcher.passedBlockNumbers = append(fetcher.passedBlockNumbers, blockNumber) + return fetcher.returnLogs, fetcher.returnErr +} + +func (fetcher *MockPriceFeedFetcher) AssertFetchLogValuesCalledWith(blockNumbers []int64) { + Expect(fetcher.passedBlockNumbers).To(Equal(blockNumbers)) +} diff --git a/pkg/transformers/test_data/mocks/price_feeds/repository.go b/pkg/transformers/test_data/mocks/price_feeds/repository.go new file mode 100644 index 00000000..cdf323e0 --- /dev/null +++ b/pkg/transformers/test_data/mocks/price_feeds/repository.go @@ -0,0 +1,49 @@ +package price_feeds + +import ( + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" +) + +type MockPriceFeedRepository struct { + createErr error + missingHeaders []core.Header + missingHeadersErr error + passedEndingBlockNumber int64 + passedModel price_feeds.PriceFeedModel + passedStartingBlockNumber int64 +} + +func (repository *MockPriceFeedRepository) SetCreateErr(err error) { + repository.createErr = err +} + +func (repository *MockPriceFeedRepository) SetMissingHeadersErr(err error) { + repository.missingHeadersErr = err +} + +func (repository *MockPriceFeedRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockPriceFeedRepository) Create(model price_feeds.PriceFeedModel) error { + repository.passedModel = model + return repository.createErr +} + +func (repository *MockPriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.passedStartingBlockNumber = startingBlockNumber + repository.passedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockPriceFeedRepository) AssertCreateCalledWith(model price_feeds.PriceFeedModel) { + Expect(repository.passedModel).To(Equal(model)) +} + +func (repository *MockPriceFeedRepository) AssertMissingHeadersCalledwith(startingBlockNumber, endingBlockNumber int64) { + Expect(repository.passedStartingBlockNumber).To(Equal(startingBlockNumber)) + Expect(repository.passedEndingBlockNumber).To(Equal(endingBlockNumber)) +} diff --git a/pkg/transformers/transformer.go b/pkg/transformers/transformer.go deleted file mode 100644 index 6c034478..00000000 --- a/pkg/transformers/transformer.go +++ /dev/null @@ -1,9 +0,0 @@ -package transformers - -import ( - "github.com/vulcanize/vulcanizedb/pkg/core" -) - -type Transformer interface { - Execute(header core.Header, headerID int64) error -} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 3290222f..e554c429 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -17,6 +17,7 @@ package transformers import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) @@ -25,8 +26,11 @@ func TransformerInitializers() []shared.TransformerInitializer { flipKickTransformerInitializer := flip_kick.FlipKickTransformerInitializer{Config: flipKickConfig} frobConfig := frob.FrobConfig frobTransformerInitializer := frob.FrobTransformerInitializer{Config: frobConfig} + priceFeedConfig := price_feeds.PriceFeedConfig + priceFeedTransformerInitializer := price_feeds.PriceFeedTransformerInitializer{Config: priceFeedConfig} return []shared.TransformerInitializer{ flipKickTransformerInitializer.NewFlipKickTransformer, frobTransformerInitializer.NewFrobTransformer, + priceFeedTransformerInitializer.NewPriceFeedTransformer, } } diff --git a/test_config/test_config.go b/test_config/test_config.go index 2d240572..e6926a82 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -77,9 +77,6 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM headers") db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") - db.MustExec("DELETE FROM maker.peps") - db.MustExec("DELETE FROM maker.pips") - db.MustExec("DELETE FROM maker.reps") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts")