From 2949996d227d14c6c9ad061407340792e6ef64d4 Mon Sep 17 00:00:00 2001 From: Rob Mulholand Date: Thu, 26 Jul 2018 13:57:38 -0500 Subject: [PATCH] Add Pip price feed --- cmd/syncPriceFeeds.go | 5 +- .../1532623709_create_pips_table.down.sql | 1 + .../1532623709_create_pips_table.up.sql | 9 +++ db/schema.sql | 55 ++++++++++++++ environments/infura.toml | 2 +- pkg/fakes/mock_blockchain.go | 4 +- pkg/transformers/pep/constants.go | 6 -- pkg/transformers/pep/pep.go | 7 -- pkg/transformers/price_feeds/constants.go | 19 +++++ .../{ => price_feeds}/pep/fetcher.go | 19 ++--- .../{ => price_feeds}/pep/fetcher_test.go | 15 ++-- .../{ => price_feeds}/pep/pep_suite_test.go | 2 +- .../{ => price_feeds}/pep/repository.go | 10 +-- .../{ => price_feeds}/pep/repository_test.go | 23 +++++- .../{ => price_feeds}/pep/transformer.go | 26 ++----- .../{ => price_feeds}/pep/transformer_test.go | 6 +- pkg/transformers/price_feeds/pip/fetcher.go | 75 +++++++++++++++++++ .../price_feeds/pip/fetcher_test.go | 60 +++++++++++++++ .../price_feeds/pip/pip_suite_test.go | 13 ++++ .../price_feeds/pip/repository.go | 25 +++++++ .../price_feeds/pip/repository_test.go | 51 +++++++++++++ .../price_feeds/pip/transformer.go | 43 +++++++++++ .../price_feeds/pip/transformer_test.go | 45 +++++++++++ pkg/transformers/price_feeds/price_update.go | 21 ++++++ test_config/test_config.go | 1 + 25 files changed, 474 insertions(+), 69 deletions(-) create mode 100644 db/migrations/1532623709_create_pips_table.down.sql create mode 100644 db/migrations/1532623709_create_pips_table.up.sql delete mode 100644 pkg/transformers/pep/constants.go delete mode 100644 pkg/transformers/pep/pep.go create mode 100644 pkg/transformers/price_feeds/constants.go rename pkg/transformers/{ => price_feeds}/pep/fetcher.go (74%) rename pkg/transformers/{ => price_feeds}/pep/fetcher_test.go (78%) rename pkg/transformers/{ => price_feeds}/pep/pep_suite_test.go (90%) rename pkg/transformers/{ => price_feeds}/pep/repository.go (68%) rename pkg/transformers/{ => price_feeds}/pep/repository_test.go (62%) rename pkg/transformers/{ => price_feeds}/pep/transformer.go (68%) rename pkg/transformers/{ => price_feeds}/pep/transformer_test.go (87%) create mode 100644 pkg/transformers/price_feeds/pip/fetcher.go create mode 100644 pkg/transformers/price_feeds/pip/fetcher_test.go create mode 100644 pkg/transformers/price_feeds/pip/pip_suite_test.go create mode 100644 pkg/transformers/price_feeds/pip/repository.go create mode 100644 pkg/transformers/price_feeds/pip/repository_test.go create mode 100644 pkg/transformers/price_feeds/pip/transformer.go create mode 100644 pkg/transformers/price_feeds/pip/transformer_test.go create mode 100644 pkg/transformers/price_feeds/price_update.go diff --git a/cmd/syncPriceFeeds.go b/cmd/syncPriceFeeds.go index f70482c0..c977f35b 100644 --- a/cmd/syncPriceFeeds.go +++ b/cmd/syncPriceFeeds.go @@ -32,7 +32,8 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/node" "github.com/vulcanize/vulcanizedb/pkg/history" "github.com/vulcanize/vulcanizedb/pkg/transformers" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pep" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" "github.com/vulcanize/vulcanizedb/utils" ) @@ -87,10 +88,12 @@ func syncPriceFeeds() { db := utils.LoadPostgres(databaseConfig, blockChain.Node()) headerRepository := repositories.NewHeaderRepository(&db) + // TODO: add transformers to validation so we don't miss events on new block headers validator := history.NewHeaderValidator(blockChain, headerRepository, validationWindow) missingBlocksPopulated := make(chan int) transformers := []transformers.Transformer{ pep.NewPepTransformer(blockChain, &db), + pip.NewPipTransformer(blockChain, &db), } go backFillPriceFeeds(blockChain, headerRepository, missingBlocksPopulated, startingBlockNumber, transformers) diff --git a/db/migrations/1532623709_create_pips_table.down.sql b/db/migrations/1532623709_create_pips_table.down.sql new file mode 100644 index 00000000..cd5c39b5 --- /dev/null +++ b/db/migrations/1532623709_create_pips_table.down.sql @@ -0,0 +1 @@ +DROP TABLE maker.pips; \ No newline at end of file diff --git a/db/migrations/1532623709_create_pips_table.up.sql b/db/migrations/1532623709_create_pips_table.up.sql new file mode 100644 index 00000000..a957e825 --- /dev/null +++ b/db/migrations/1532623709_create_pips_table.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE maker.pips ( + id SERIAL PRIMARY KEY, + block_number BIGINT NOT NULL, + header_id INTEGER NOT NULL, + usd_value NUMERIC, + CONSTRAINT headers_fk FOREIGN KEY (header_id) + REFERENCES headers (id) + ON DELETE CASCADE +); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 9b89e19b..461507e3 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -151,6 +151,38 @@ CREATE SEQUENCE maker.peps_id_seq ALTER SEQUENCE maker.peps_id_seq OWNED BY maker.peps.id; +-- +-- Name: pips; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.pips ( + id integer NOT NULL, + block_number bigint NOT NULL, + header_id integer NOT NULL, + usd_value numeric +); + + +-- +-- Name: pips_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.pips_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: pips_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.pips_id_seq OWNED BY maker.pips.id; + + -- -- Name: logs; Type: TABLE; Schema: public; Owner: - -- @@ -544,6 +576,13 @@ ALTER TABLE ONLY maker.frob ALTER COLUMN id SET DEFAULT nextval('maker.frob_id_s ALTER TABLE ONLY maker.peps ALTER COLUMN id SET DEFAULT nextval('maker.peps_id_seq'::regclass); +-- +-- Name: pips id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pips ALTER COLUMN id SET DEFAULT nextval('maker.pips_id_seq'::regclass); + + -- -- Name: blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -647,6 +686,14 @@ ALTER TABLE ONLY maker.peps ADD CONSTRAINT peps_pkey PRIMARY KEY (id); +-- +-- Name: pips pips_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pips + ADD CONSTRAINT pips_pkey PRIMARY KEY (id); + + -- -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -802,6 +849,14 @@ ALTER TABLE ONLY maker.peps ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: pips headers_fk; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.pips + ADD CONSTRAINT headers_fk FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: transactions blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: - -- diff --git a/environments/infura.toml b/environments/infura.toml index bd273b20..e1539f3d 100644 --- a/environments/infura.toml +++ b/environments/infura.toml @@ -1,5 +1,5 @@ [database] -name = "vulcanize_private" +name = "vulcanize_public" hostname = "localhost" port = 5432 diff --git a/pkg/fakes/mock_blockchain.go b/pkg/fakes/mock_blockchain.go index 1523a39f..4869bdae 100644 --- a/pkg/fakes/mock_blockchain.go +++ b/pkg/fakes/mock_blockchain.go @@ -65,7 +65,7 @@ func (chain *MockBlockChain) SetGetLogsReturnLogs(logs []core.Log) { chain.getLogsReturnLogs = logs } -func (chain *MockBlockChain) FetchContractData(abiJSON string, address string, method string, methodArg interface{}, result interface{}, blockNumber int64) error { +func (chain *MockBlockChain) FetchContractData(abiJSON, address, method string, methodArg, result interface{}, blockNumber int64) error { chain.fetchContractDataPassedAbi = abiJSON chain.fetchContractDataPassedAddress = address chain.fetchContractDataPassedMethod = method @@ -114,7 +114,7 @@ func (chain *MockBlockChain) AssertFetchContractDataCalledWith(abiJSON string, a if methodArg != nil { Expect(chain.fetchContractDataPassedMethodArg).To(Equal(methodArg)) } - Expect(chain.fetchContractDataPassedResult).To(Equal(result)) + Expect(chain.fetchContractDataPassedResult).To(BeAssignableToTypeOf(result)) Expect(chain.fetchContractDataPassedBlockNumber).To(Equal(blockNumber)) } diff --git a/pkg/transformers/pep/constants.go b/pkg/transformers/pep/constants.go deleted file mode 100644 index ecd0160c..00000000 --- a/pkg/transformers/pep/constants.go +++ /dev/null @@ -1,6 +0,0 @@ -package pep - -var ( - LogValueTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527" - PepAddress = "0x99041f808d598b782d5a3e498681c2452a31da08" -) diff --git a/pkg/transformers/pep/pep.go b/pkg/transformers/pep/pep.go deleted file mode 100644 index f652b420..00000000 --- a/pkg/transformers/pep/pep.go +++ /dev/null @@ -1,7 +0,0 @@ -package pep - -type Pep struct { - BlockNumber int64 `db:"block_number"` - HeaderID int64 `db:"header_id"` - UsdValue string `db:"usd_value"` -} diff --git a/pkg/transformers/price_feeds/constants.go b/pkg/transformers/price_feeds/constants.go new file mode 100644 index 00000000..480cc314 --- /dev/null +++ b/pkg/transformers/price_feeds/constants.go @@ -0,0 +1,19 @@ +package price_feeds + +import ( + "errors" + "math/big" +) + +var ( + ErrMultipleLogs = errors.New("multiple matching logs found in block") + ErrNoMatchingLog = errors.New("no matching log") + PeekMethodName = "peek" + PepLogTopic0 = "0x296ba4ca62c6c21c95e828080cb8aec7481b71390585605300a8a76f9e95b527" + PipLogTopic0 = "0x1817835800000000000000000000000000000000000000000000000000000000" + PepAddress = "0x99041F808D598B782D5a3e498681C2452A31da08" + PipAddress = "0x729D19f657BD0614b4985Cf1D82531c67569197B" + PipMedianizerABI = `[{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"","type":"bytes32"}],"name":"poke","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"poke","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"compute","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"wat","type":"address"}],"name":"unset","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"indexes","outputs":[{"name":"","type":"bytes12"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"next","outputs":[{"name":"","type":"bytes12"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"read","outputs":[{"name":"","type":"bytes32"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"peek","outputs":[{"name":"","type":"bytes32"},{"name":"","type":"bool"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"","type":"bytes12"}],"name":"values","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"min_","type":"uint96"}],"name":"setMin","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[],"name":"void","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"},{"name":"wat","type":"address"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"pos","type":"bytes12"}],"name":"unset","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"next_","type":"bytes12"}],"name":"setNext","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"min","outputs":[{"name":"","type":"uint96"}],"payable":false,"type":"function"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"}]` + Ether = big.NewFloat(1e18) + Ray = big.NewFloat(1e27) +) diff --git a/pkg/transformers/pep/fetcher.go b/pkg/transformers/price_feeds/pep/fetcher.go similarity index 74% rename from pkg/transformers/pep/fetcher.go rename to pkg/transformers/price_feeds/pep/fetcher.go index 0a014744..827cb0d3 100644 --- a/pkg/transformers/pep/fetcher.go +++ b/pkg/transformers/price_feeds/pep/fetcher.go @@ -1,17 +1,14 @@ package pep import ( - "errors" + "math/big" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/core" - "math/big" -) -var ( - ErrMultipleLogs = errors.New("multiple matching logs found in block") - ErrNoMatchingLog = errors.New("no matching log") + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" ) type IPepFetcher interface { @@ -33,8 +30,8 @@ func (fetcher PepFetcher) FetchPepValue(header core.Header) (string, error) { query := ethereum.FilterQuery{ FromBlock: blockNumber, ToBlock: blockNumber, - Addresses: []common.Address{common.HexToAddress(PepAddress)}, - Topics: [][]common.Hash{{common.HexToHash(LogValueTopic0)}}, + Addresses: []common.Address{common.HexToAddress(price_feeds.PepAddress)}, + Topics: [][]common.Hash{{common.HexToHash(price_feeds.PepLogTopic0)}}, } logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) return fetcher.getLogValue(logs, err) @@ -45,10 +42,10 @@ func (fetcher PepFetcher) getLogValue(logs []types.Log, err error) (string, erro return "", err } if len(logs) < 1 { - return "", ErrNoMatchingLog + return "", price_feeds.ErrNoMatchingLog } if len(logs) > 1 { - return "", ErrMultipleLogs + return "", price_feeds.ErrMultipleLogs } return string(logs[0].Data), nil } diff --git a/pkg/transformers/pep/fetcher_test.go b/pkg/transformers/price_feeds/pep/fetcher_test.go similarity index 78% rename from pkg/transformers/pep/fetcher_test.go rename to pkg/transformers/price_feeds/pep/fetcher_test.go index b1dd498f..209edef0 100644 --- a/pkg/transformers/pep/fetcher_test.go +++ b/pkg/transformers/price_feeds/pep/fetcher_test.go @@ -9,12 +9,13 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pep" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" "math/big" ) var _ = Describe("Pep fetcher", func() { - It("calls contract to peek mkr/usd value", func() { + It("gets logs describing updated mkr/usd value", func() { mockBlockChain := fakes.NewMockBlockChain() mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}}) fetcher := pep.NewPepFetcher(mockBlockChain) @@ -31,13 +32,13 @@ var _ = Describe("Pep fetcher", func() { expectedQuery := ethereum.FilterQuery{ FromBlock: big.NewInt(blockNumber), ToBlock: big.NewInt(blockNumber), - Addresses: []common.Address{common.HexToAddress(pep.PepAddress)}, - Topics: [][]common.Hash{{common.HexToHash(pep.LogValueTopic0)}}, + Addresses: []common.Address{common.HexToAddress(price_feeds.PepAddress)}, + Topics: [][]common.Hash{{common.HexToHash(price_feeds.PepLogTopic0)}}, } mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) }) - It("returns error if contract call fails", func() { + It("returns error if getting logs fails", func() { mockBlockChain := fakes.NewMockBlockChain() mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{}}) mockBlockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) @@ -56,7 +57,7 @@ var _ = Describe("Pep fetcher", func() { _, err := fetcher.FetchPepValue(core.Header{}) Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(pep.ErrNoMatchingLog)) + Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog)) }) It("returns error if more than one matching logs returned", func() { @@ -67,6 +68,6 @@ var _ = Describe("Pep fetcher", func() { _, err := fetcher.FetchPepValue(core.Header{}) Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(pep.ErrMultipleLogs)) + Expect(err).To(MatchError(price_feeds.ErrMultipleLogs)) }) }) diff --git a/pkg/transformers/pep/pep_suite_test.go b/pkg/transformers/price_feeds/pep/pep_suite_test.go similarity index 90% rename from pkg/transformers/pep/pep_suite_test.go rename to pkg/transformers/price_feeds/pep/pep_suite_test.go index 1a885590..49dcb4cd 100644 --- a/pkg/transformers/pep/pep_suite_test.go +++ b/pkg/transformers/price_feeds/pep/pep_suite_test.go @@ -1,4 +1,4 @@ -package pep_test +package pep import ( "testing" diff --git a/pkg/transformers/pep/repository.go b/pkg/transformers/price_feeds/pep/repository.go similarity index 68% rename from pkg/transformers/pep/repository.go rename to pkg/transformers/price_feeds/pep/repository.go index 65afba49..27832e0e 100644 --- a/pkg/transformers/pep/repository.go +++ b/pkg/transformers/price_feeds/pep/repository.go @@ -2,10 +2,11 @@ package pep import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" ) type IPepRepository interface { - CreatePep(pep Pep) error + CreatePep(pep price_feeds.PriceUpdate) error } type PepRepository struct { @@ -18,10 +19,7 @@ func NewPepRepository(db *postgres.DB) PepRepository { } } -func (repository PepRepository) CreatePep(pep Pep) error { +func (repository PepRepository) CreatePep(pep price_feeds.PriceUpdate) error { _, err := repository.db.Exec(`INSERT INTO maker.peps (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, pep.BlockNumber, pep.HeaderID, pep.UsdValue) - if err != nil { - return err - } - return nil + return err } diff --git a/pkg/transformers/pep/repository_test.go b/pkg/transformers/price_feeds/pep/repository_test.go similarity index 62% rename from pkg/transformers/pep/repository_test.go rename to pkg/transformers/price_feeds/pep/repository_test.go index 85393b3d..0a092e27 100644 --- a/pkg/transformers/pep/repository_test.go +++ b/pkg/transformers/price_feeds/pep/repository_test.go @@ -6,12 +6,27 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pep" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("Pep repository", func() { - It("creates a pep", func() { + It("returns header if matching header does not exist", func() { + db := test_config.NewTestDB(core.Node{}) + repository := pep.NewPepRepository(db) + pepToAdd := price_feeds.PriceUpdate{ + BlockNumber: 0, + HeaderID: 0, + UsdValue: "123.456", + } + + err := repository.CreatePep(pepToAdd) + + Expect(err).To(HaveOccurred()) + }) + + It("creates a pep when matching header exists", func() { db := test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) repository := pep.NewPepRepository(db) @@ -19,7 +34,7 @@ var _ = Describe("Pep repository", func() { headerRepository := repositories.NewHeaderRepository(db) headerID, err := headerRepository.CreateOrUpdateHeader(header) Expect(err).NotTo(HaveOccurred()) - pepToAdd := pep.Pep{ + pepToAdd := price_feeds.PriceUpdate{ BlockNumber: header.BlockNumber, HeaderID: headerID, UsdValue: "123.456", @@ -28,7 +43,7 @@ var _ = Describe("Pep repository", func() { err = repository.CreatePep(pepToAdd) Expect(err).NotTo(HaveOccurred()) - var dbPep pep.Pep + var dbPep price_feeds.PriceUpdate err = db.Get(&dbPep, `SELECT block_number, header_id, usd_value FROM maker.peps WHERE header_id = $1`, pepToAdd.HeaderID) Expect(err).NotTo(HaveOccurred()) Expect(dbPep).To(Equal(pepToAdd)) diff --git a/pkg/transformers/pep/transformer.go b/pkg/transformers/price_feeds/pep/transformer.go similarity index 68% rename from pkg/transformers/pep/transformer.go rename to pkg/transformers/price_feeds/pep/transformer.go index bffd8814..c108cbca 100644 --- a/pkg/transformers/pep/transformer.go +++ b/pkg/transformers/price_feeds/pep/transformer.go @@ -1,15 +1,11 @@ package pep import ( - "math/big" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" ) -var Ether = big.NewFloat(1e18) -var Ray = big.NewFloat(1e27) - type PepTransformer struct { fetcher IPepFetcher repository IPepRepository @@ -27,7 +23,7 @@ func NewPepTransformer(chain core.BlockChain, db *postgres.DB) PepTransformer { func (transformer PepTransformer) Execute(header core.Header, headerID int64) error { logValue, err := transformer.fetcher.FetchPepValue(header) if err != nil { - if err == ErrNoMatchingLog { + if err == price_feeds.ErrNoMatchingLog { return nil } return err @@ -36,24 +32,12 @@ func (transformer PepTransformer) Execute(header core.Header, headerID int64) er return transformer.repository.CreatePep(pep) } -func getPep(logValue string, header core.Header, headerID int64) Pep { - valueInUSD := convert("wad", logValue, 15) - pep := Pep{ +func getPep(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate { + valueInUSD := price_feeds.Convert("wad", logValue, 15) + pep := price_feeds.PriceUpdate{ BlockNumber: header.BlockNumber, HeaderID: headerID, UsdValue: valueInUSD, } return pep } - -func convert(conversion string, value string, prec int) string { - var bgflt = big.NewFloat(0.0) - bgflt.SetString(value) - switch conversion { - case "ray": - bgflt.Quo(bgflt, Ray) - case "wad": - bgflt.Quo(bgflt, Ether) - } - return bgflt.Text('g', prec) -} diff --git a/pkg/transformers/pep/transformer_test.go b/pkg/transformers/price_feeds/pep/transformer_test.go similarity index 87% rename from pkg/transformers/pep/transformer_test.go rename to pkg/transformers/price_feeds/pep/transformer_test.go index 220774e4..ec6d6d9c 100644 --- a/pkg/transformers/pep/transformer_test.go +++ b/pkg/transformers/price_feeds/pep/transformer_test.go @@ -8,7 +8,8 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pep" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pep" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -27,6 +28,7 @@ var _ = Describe("Pep transformer", func() { chain := fakes.NewMockBlockChain() chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}}) db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) headerRepository := repositories.NewHeaderRepository(db) header := core.Header{BlockNumber: 12345} headerID, err := headerRepository.CreateOrUpdateHeader(header) @@ -36,7 +38,7 @@ var _ = Describe("Pep transformer", func() { err = transformer.Execute(header, headerID) Expect(err).NotTo(HaveOccurred()) - var dbPep pep.Pep + var dbPep price_feeds.PriceUpdate err = db.Get(&dbPep, `SELECT block_number, header_id, usd_value FROM maker.peps WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) Expect(dbPep.BlockNumber).To(Equal(header.BlockNumber)) diff --git a/pkg/transformers/price_feeds/pip/fetcher.go b/pkg/transformers/price_feeds/pip/fetcher.go new file mode 100644 index 00000000..849cc147 --- /dev/null +++ b/pkg/transformers/price_feeds/pip/fetcher.go @@ -0,0 +1,75 @@ +package pip + +import ( + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "math/big" +) + +type IPipFetcher interface { + FetchPipValue(header core.Header) (string, error) +} + +type PipFetcher struct { + blockChain core.BlockChain +} + +func NewPipFetcher(chain core.BlockChain) PipFetcher { + return PipFetcher{ + blockChain: chain, + } +} + +func (fetcher PipFetcher) FetchPipValue(header core.Header) (string, error) { + blockNumber := big.NewInt(header.BlockNumber) + query := ethereum.FilterQuery{ + FromBlock: blockNumber, + ToBlock: blockNumber, + Addresses: []common.Address{common.HexToAddress(price_feeds.PipAddress)}, + Topics: [][]common.Hash{{common.HexToHash(price_feeds.PipLogTopic0)}}, + } + logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) + if err != nil { + return "", err + } + if len(logs) > 0 { + return fetcher.getLogValue(logs, err) + } + return "", price_feeds.ErrNoMatchingLog +} + +func (fetcher PipFetcher) getLogValue(logs []types.Log, err error) (string, error) { + var ( + ret0 = new([32]byte) + ret1 = new(bool) + ) + var r = &[]interface{}{ + ret0, + ret1, + } + err = fetcher.blockChain.FetchContractData(price_feeds.PipMedianizerABI, price_feeds.PipAddress, price_feeds.PeekMethodName, nil, r, int64(logs[0].BlockNumber)) + if err != nil { + return "", err + } + result := newResult(*ret0, *ret1) + return result.Value.String(), nil +} + +type Value [32]byte + +type Peek struct { + Value + OK bool +} + +func (value Value) String() string { + bi := big.NewInt(0).SetBytes(value[:]) + return bi.String() +} + +func newResult(value [32]byte, ok bool) *Peek { + return &Peek{Value: value, OK: ok} +} diff --git a/pkg/transformers/price_feeds/pip/fetcher_test.go b/pkg/transformers/price_feeds/pip/fetcher_test.go new file mode 100644 index 00000000..c5acdc69 --- /dev/null +++ b/pkg/transformers/price_feeds/pip/fetcher_test.go @@ -0,0 +1,60 @@ +package pip_test + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" +) + +var _ = Describe("Pip fetcher", func() { + It("returns error if fetching logs fails", func() { + chain := fakes.NewMockBlockChain() + chain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) + fetcher := pip.NewPipFetcher(chain) + + _, err := fetcher.FetchPipValue(core.Header{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("returns no matching logs error if no logs returned", func() { + chain := fakes.NewMockBlockChain() + fetcher := pip.NewPipFetcher(chain) + + _, err := fetcher.FetchPipValue(core.Header{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(price_feeds.ErrNoMatchingLog)) + }) + + Describe("when matching log found", func() { + It("calls contract to peek current eth/usd value", func() { + blockNumber := uint64(12345) + chain := fakes.NewMockBlockChain() + chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{BlockNumber: blockNumber}}) + fetcher := pip.NewPipFetcher(chain) + + _, err := fetcher.FetchPipValue(core.Header{}) + + Expect(err).NotTo(HaveOccurred()) + chain.AssertFetchContractDataCalledWith(price_feeds.PipMedianizerABI, price_feeds.PipAddress, price_feeds.PeekMethodName, nil, &[]interface{}{[32]byte{}, false}, int64(blockNumber)) + }) + + It("returns error if contract call fails", func() { + chain := fakes.NewMockBlockChain() + chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{BlockNumber: uint64(12345)}}) + chain.SetFetchContractDataErr(fakes.FakeError) + fetcher := pip.NewPipFetcher(chain) + + _, err := fetcher.FetchPipValue(core.Header{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) +}) diff --git a/pkg/transformers/price_feeds/pip/pip_suite_test.go b/pkg/transformers/price_feeds/pip/pip_suite_test.go new file mode 100644 index 00000000..30668aa9 --- /dev/null +++ b/pkg/transformers/price_feeds/pip/pip_suite_test.go @@ -0,0 +1,13 @@ +package pip + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPip(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Pip Suite") +} diff --git a/pkg/transformers/price_feeds/pip/repository.go b/pkg/transformers/price_feeds/pip/repository.go new file mode 100644 index 00000000..b6f3edc5 --- /dev/null +++ b/pkg/transformers/price_feeds/pip/repository.go @@ -0,0 +1,25 @@ +package pip + +import ( + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" +) + +type IPipRepository interface { + CreatePip(pip price_feeds.PriceUpdate) error +} + +type PipRepository struct { + db *postgres.DB +} + +func NewPipRepository(db *postgres.DB) PipRepository { + return PipRepository{ + db: db, + } +} + +func (repository PipRepository) CreatePip(pip price_feeds.PriceUpdate) error { + _, err := repository.db.Exec(`INSERT INTO maker.pips (block_number, header_id, usd_value) VALUES ($1, $2, $3::NUMERIC)`, pip.BlockNumber, pip.HeaderID, pip.UsdValue) + return err +} diff --git a/pkg/transformers/price_feeds/pip/repository_test.go b/pkg/transformers/price_feeds/pip/repository_test.go new file mode 100644 index 00000000..c47c0e36 --- /dev/null +++ b/pkg/transformers/price_feeds/pip/repository_test.go @@ -0,0 +1,51 @@ +package pip_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Pip repository", func() { + It("does not create a pip if no matching header", func() { + db := test_config.NewTestDB(core.Node{}) + repository := pip.NewPipRepository(db) + priceUpdate := price_feeds.PriceUpdate{ + BlockNumber: 0, + HeaderID: 0, + UsdValue: "123", + } + + err := repository.CreatePip(priceUpdate) + + Expect(err).To(HaveOccurred()) + }) + + It("creates a pip when header exists", func() { + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + repository := pip.NewPipRepository(db) + headerRepository := repositories.NewHeaderRepository(db) + header := core.Header{BlockNumber: 12345} + headerID, err := headerRepository.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + priceUpdate := price_feeds.PriceUpdate{ + BlockNumber: header.BlockNumber, + HeaderID: headerID, + UsdValue: "777.777", + } + + err = repository.CreatePip(priceUpdate) + + Expect(err).NotTo(HaveOccurred()) + var dbPip price_feeds.PriceUpdate + err = db.Get(&dbPip, `SELECT block_number, header_id, usd_value FROM maker.pips WHERE block_number = $1`, header.BlockNumber) + Expect(err).NotTo(HaveOccurred()) + Expect(dbPip).To(Equal(priceUpdate)) + }) +}) diff --git a/pkg/transformers/price_feeds/pip/transformer.go b/pkg/transformers/price_feeds/pip/transformer.go new file mode 100644 index 00000000..8b66b4dd --- /dev/null +++ b/pkg/transformers/price_feeds/pip/transformer.go @@ -0,0 +1,43 @@ +package pip + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" +) + +type PipTransformer struct { + fetcher IPipFetcher + repository IPipRepository +} + +func NewPipTransformer(chain core.BlockChain, db *postgres.DB) PipTransformer { + fetcher := NewPipFetcher(chain) + repository := NewPipRepository(db) + return PipTransformer{ + fetcher: fetcher, + repository: repository, + } +} + +func (transformer PipTransformer) Execute(header core.Header, headerID int64) error { + value, err := transformer.fetcher.FetchPipValue(header) + if err != nil { + if err == price_feeds.ErrNoMatchingLog { + return nil + } + return err + } + pip := getPip(value, header, headerID) + return transformer.repository.CreatePip(pip) +} + +func getPip(logValue string, header core.Header, headerID int64) price_feeds.PriceUpdate { + valueInUSD := price_feeds.Convert("wad", logValue, 15) + pep := price_feeds.PriceUpdate{ + BlockNumber: header.BlockNumber, + HeaderID: headerID, + UsdValue: valueInUSD, + } + return pep +} diff --git a/pkg/transformers/price_feeds/pip/transformer_test.go b/pkg/transformers/price_feeds/pip/transformer_test.go new file mode 100644 index 00000000..729de585 --- /dev/null +++ b/pkg/transformers/price_feeds/pip/transformer_test.go @@ -0,0 +1,45 @@ +package pip_test + +import ( + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" + "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds/pip" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Pip transformer", func() { + It("returns nil if no logs found", func() { + chain := fakes.NewMockBlockChain() + db := test_config.NewTestDB(core.Node{}) + transformer := pip.NewPipTransformer(chain, db) + + err := transformer.Execute(core.Header{}, 123) + + Expect(err).NotTo(HaveOccurred()) + }) + + It("creates pip row for found log", func() { + chain := fakes.NewMockBlockChain() + chain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{{Data: []byte{1, 2, 3, 4, 5}}}) + db := test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + header := core.Header{BlockNumber: 12345} + headerID, err := headerRepository.CreateOrUpdateHeader(header) + Expect(err).NotTo(HaveOccurred()) + transformer := pip.NewPipTransformer(chain, db) + + err = transformer.Execute(header, headerID) + + Expect(err).NotTo(HaveOccurred()) + var dbPip price_feeds.PriceUpdate + err = db.Get(&dbPip, `SELECT block_number, header_id, usd_value FROM maker.pips WHERE header_id = $1`, headerID) + Expect(err).NotTo(HaveOccurred()) + Expect(dbPip.BlockNumber).To(Equal(header.BlockNumber)) + }) +}) diff --git a/pkg/transformers/price_feeds/price_update.go b/pkg/transformers/price_feeds/price_update.go new file mode 100644 index 00000000..cd70066a --- /dev/null +++ b/pkg/transformers/price_feeds/price_update.go @@ -0,0 +1,21 @@ +package price_feeds + +import "math/big" + +type PriceUpdate struct { + BlockNumber int64 `db:"block_number"` + HeaderID int64 `db:"header_id"` + UsdValue string `db:"usd_value"` +} + +func Convert(conversion string, value string, prec int) string { + var bgflt = big.NewFloat(0.0) + bgflt.SetString(value) + switch conversion { + case "ray": + bgflt.Quo(bgflt, Ray) + case "wad": + bgflt.Quo(bgflt, Ether) + } + return bgflt.Text('g', prec) +} diff --git a/test_config/test_config.go b/test_config/test_config.go index eafce6f0..845cd00f 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -78,6 +78,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM log_filters") db.MustExec("DELETE FROM logs") db.MustExec("DELETE FROM maker.peps") + db.MustExec("DELETE FROM maker.pips") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts")