From 4d99e8a3248b1c5d087e1bd3e6805d6940b30c62 Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Thu, 11 Oct 2018 12:16:10 -0500 Subject: [PATCH 1/6] migration for vat flux --- .../1539276289_create_vat_flux_table.down.sql | 3 + .../1539276289_create_vat_flux_table.up.sql | 14 ++++ db/schema.sql | 70 ++++++++++++++++++- 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 db/migrations/1539276289_create_vat_flux_table.down.sql create mode 100644 db/migrations/1539276289_create_vat_flux_table.up.sql diff --git a/db/migrations/1539276289_create_vat_flux_table.down.sql b/db/migrations/1539276289_create_vat_flux_table.down.sql new file mode 100644 index 00000000..c42423f1 --- /dev/null +++ b/db/migrations/1539276289_create_vat_flux_table.down.sql @@ -0,0 +1,3 @@ +DROP TABLE maker.vat_flux; +ALTER TABLE public.checked_headers + DROP COLUMN vat_flux_checked; diff --git a/db/migrations/1539276289_create_vat_flux_table.up.sql b/db/migrations/1539276289_create_vat_flux_table.up.sql new file mode 100644 index 00000000..9504734c --- /dev/null +++ b/db/migrations/1539276289_create_vat_flux_table.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE maker.vat_flux ( + id SERIAL PRIMARY KEY, + header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, + ilk TEXT, + src TEXT, + dst TEXT, + rad numeric, + tx_idx INTEGER NOT NULL, + raw_log JSONB, + UNIQUE (header_id, tx_idx) +); + +ALTER TABLE public.checked_headers + ADD COLUMN vat_flux_checked BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/db/schema.sql b/db/schema.sql index e6e00893..bd360551 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -697,6 +697,42 @@ CREATE SEQUENCE maker.tend_id_seq ALTER SEQUENCE maker.tend_id_seq OWNED BY maker.tend.id; +-- +-- Name: vat_flux; Type: TABLE; Schema: maker; Owner: - +-- + +CREATE TABLE maker.vat_flux ( + id integer NOT NULL, + header_id integer NOT NULL, + ilk text, + src text, + dst text, + rad numeric, + tx_idx integer NOT NULL, + raw_log jsonb +); + + +-- +-- Name: vat_flux_id_seq; Type: SEQUENCE; Schema: maker; Owner: - +-- + +CREATE SEQUENCE maker.vat_flux_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: vat_flux_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: - +-- + +ALTER SEQUENCE maker.vat_flux_id_seq OWNED BY maker.vat_flux.id; + + -- -- Name: vat_fold; Type: TABLE; Schema: maker; Owner: - -- @@ -1052,7 +1088,8 @@ CREATE TABLE public.checked_headers ( vat_heal_checked boolean DEFAULT false NOT NULL, vat_toll_checked boolean DEFAULT false NOT NULL, vat_tune_checked boolean DEFAULT false NOT NULL, - vat_grab_checked boolean DEFAULT false NOT NULL + vat_grab_checked boolean DEFAULT false NOT NULL, + vat_flux_checked boolean DEFAULT false NOT NULL ); @@ -1500,6 +1537,13 @@ ALTER TABLE ONLY maker.price_feeds ALTER COLUMN id SET DEFAULT nextval('maker.pr ALTER TABLE ONLY maker.tend ALTER COLUMN id SET DEFAULT nextval('maker.tend_id_seq'::regclass); +-- +-- Name: vat_flux id; Type: DEFAULT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_flux ALTER COLUMN id SET DEFAULT nextval('maker.vat_flux_id_seq'::regclass); + + -- -- Name: vat_fold id; Type: DEFAULT; Schema: maker; Owner: - -- @@ -1907,6 +1951,22 @@ ALTER TABLE ONLY maker.tend ADD CONSTRAINT tend_pkey PRIMARY KEY (id); +-- +-- Name: vat_flux vat_flux_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_flux + ADD CONSTRAINT vat_flux_header_id_tx_idx_key UNIQUE (header_id, tx_idx); + + +-- +-- Name: vat_flux vat_flux_pkey; Type: CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_flux + ADD CONSTRAINT vat_flux_pkey PRIMARY KEY (id); + + -- -- Name: vat_fold vat_fold_header_id_tx_idx_key; Type: CONSTRAINT; Schema: maker; Owner: - -- @@ -2317,6 +2377,14 @@ ALTER TABLE ONLY maker.tend ADD CONSTRAINT tend_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; +-- +-- Name: vat_flux vat_flux_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - +-- + +ALTER TABLE ONLY maker.vat_flux + ADD CONSTRAINT vat_flux_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE; + + -- -- Name: vat_fold vat_fold_header_id_fkey; Type: FK CONSTRAINT; Schema: maker; Owner: - -- From 2e72a7e940c38dc90dab98b252dc6a41d082df6b Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Wed, 17 Oct 2018 10:53:06 -0500 Subject: [PATCH 2/6] vat flux transformer --- cmd/continuousLogSync.go | 1 + pkg/transformers/shared/constants.go | 5 +- .../shared/event_signature_generator_test.go | 7 + .../test_data/mocks/vat_flux/converter.go | 22 ++ .../test_data/mocks/vat_flux/repository.go | 65 +++++ pkg/transformers/test_data/vat_flux.go | 52 ++++ pkg/transformers/transformers.go | 3 + pkg/transformers/vat_flux/config.go | 11 + pkg/transformers/vat_flux/converter.go | 75 ++++++ pkg/transformers/vat_flux/converter_test.go | 54 +++++ pkg/transformers/vat_flux/model.go | 24 ++ pkg/transformers/vat_flux/repository.go | 84 +++++++ pkg/transformers/vat_flux/repository_test.go | 226 ++++++++++++++++++ pkg/transformers/vat_flux/transformer.go | 63 +++++ pkg/transformers/vat_flux/transformer_test.go | 178 ++++++++++++++ .../vat_flux/vat_flux_suite_test.go | 19 ++ pkg/transformers/vat_grab/tranformer.go | 2 +- test_config/test_config.go | 1 + 18 files changed, 890 insertions(+), 2 deletions(-) create mode 100644 pkg/transformers/test_data/mocks/vat_flux/converter.go create mode 100644 pkg/transformers/test_data/mocks/vat_flux/repository.go create mode 100644 pkg/transformers/test_data/vat_flux.go create mode 100644 pkg/transformers/vat_flux/config.go create mode 100644 pkg/transformers/vat_flux/converter.go create mode 100644 pkg/transformers/vat_flux/converter_test.go create mode 100644 pkg/transformers/vat_flux/model.go create mode 100644 pkg/transformers/vat_flux/repository.go create mode 100644 pkg/transformers/vat_flux/repository_test.go create mode 100644 pkg/transformers/vat_flux/transformer.go create mode 100644 pkg/transformers/vat_flux/transformer_test.go create mode 100644 pkg/transformers/vat_flux/vat_flux_suite_test.go diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index fefd665d..c3138173 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -111,6 +111,7 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer + transformerInitializerMap["vatFlux"] = transformers.VatFluxTransformerInitializer return transformerInitializerMap } diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index 40234ff6..df43dc7c 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -34,7 +34,8 @@ var ( PipContractAddress = "0x9FfFE440258B79c5d6604001674A4722FfC0f7Bc" PitContractAddress = "0xe7cf3198787c9a4daac73371a38f29aaeeced87e" RepContractAddress = "0xf88bbdc1e2718f8857f30a180076ec38d53cf296" - VatContractAddress = "0xcd726790550afcd77e9a7a47e86a3f9010af126b" + //VatContractAddress = "0xcd726790550afcd77e9a7a47e86a3f9010af126b" + VatContractAddress = "0x239e6f0ab02713f1f8aa90ebeded9fc66dc96cd6" //TODO: get pit and drip file method signatures directly from the ABI biteMethod = GetSolidityMethodSignature(CatABI, "Bite") @@ -62,6 +63,7 @@ var ( vatFoldMethod = GetSolidityMethodSignature(VatABI, "fold") vatTollMethod = GetSolidityMethodSignature(VatABI, "toll") vatTuneMethod = GetSolidityMethodSignature(VatABI, "tune") + vatFluxMethod = GetSolidityMethodSignature(VatABI, "flux") BiteSignature = GetEventSignature(biteMethod) DealSignature = GetLogNoteSignature(dealMethod) @@ -88,4 +90,5 @@ var ( VatFoldSignature = GetLogNoteSignature(vatFoldMethod) VatTollSignature = GetLogNoteSignature(vatTollMethod) VatTuneSignature = GetLogNoteSignature(vatTuneMethod) + VatFluxSignature = GetLogNoteSignature(vatFluxMethod) ) diff --git a/pkg/transformers/shared/event_signature_generator_test.go b/pkg/transformers/shared/event_signature_generator_test.go index 7c96cf99..44be477d 100644 --- a/pkg/transformers/shared/event_signature_generator_test.go +++ b/pkg/transformers/shared/event_signature_generator_test.go @@ -133,6 +133,13 @@ var _ = Describe("Event signature generator", func() { Expect(expected).To(Equal(actual)) }) + It("gets the vat flux method signature", func() { + expected := "flux(bytes32,bytes32,bytes32,int256)" + actual := shared.GetSolidityMethodSignature(shared.VatABI, "flux") + + Expect(expected).To(Equal(actual)) + }) + It("gets the flip deal method signature", func() { expected := "deal(uint256)" actual := shared.GetSolidityMethodSignature(shared.FlipperABI, "deal") diff --git a/pkg/transformers/test_data/mocks/vat_flux/converter.go b/pkg/transformers/test_data/mocks/vat_flux/converter.go new file mode 100644 index 00000000..c8cfd15a --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_flux/converter.go @@ -0,0 +1,22 @@ +package vat_flux + +import ( + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +type MockVatFlux struct { + err error + PassedLogs []types.Log +} + +func (converter *MockVatFlux) ToModels(ethLogs []types.Log) ([]vat_flux.VatFluxModel, error) { + converter.PassedLogs = ethLogs + return []vat_flux.VatFluxModel{test_data.VatFluxModel}, converter.err +} + +func (converter *MockVatFlux) SetConverterError(e error) { + converter.err = e +} diff --git a/pkg/transformers/test_data/mocks/vat_flux/repository.go b/pkg/transformers/test_data/mocks/vat_flux/repository.go new file mode 100644 index 00000000..332ba278 --- /dev/null +++ b/pkg/transformers/test_data/mocks/vat_flux/repository.go @@ -0,0 +1,65 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +type MockVatFluxRepository struct { + createErr error + markHeaderCheckedErr error + MarkHeaderCheckedPassedHeaderID int64 + missingHeaders []core.Header + missingHeadersErr error + PassedStartingBlockNumber int64 + PassedEndingBlockNumber int64 + PassedHeaderID int64 + PassedModels []vat_flux.VatFluxModel +} + +func (repository *MockVatFluxRepository) MarkCheckedHeader(headerId int64) error { + repository.MarkHeaderCheckedPassedHeaderID = headerId + return repository.markHeaderCheckedErr +} + +func (repository *MockVatFluxRepository) Create(headerID int64, models []vat_flux.VatFluxModel) error { + repository.PassedHeaderID = headerID + repository.PassedModels = models + return repository.createErr +} + +func (repository *MockVatFluxRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { + repository.PassedStartingBlockNumber = startingBlockNumber + repository.PassedEndingBlockNumber = endingBlockNumber + return repository.missingHeaders, repository.missingHeadersErr +} + +func (repository *MockVatFluxRepository) SetMarkHeaderCheckedErr(e error) { + repository.markHeaderCheckedErr = e +} + +func (repository *MockVatFluxRepository) SetMissingHeadersErr(e error) { + repository.missingHeadersErr = e +} + +func (repository *MockVatFluxRepository) SetMissingHeaders(headers []core.Header) { + repository.missingHeaders = headers +} + +func (repository *MockVatFluxRepository) SetCreateError(e error) { + repository.createErr = e +} diff --git a/pkg/transformers/test_data/vat_flux.go b/pkg/transformers/test_data/vat_flux.go new file mode 100644 index 00000000..76dc4f49 --- /dev/null +++ b/pkg/transformers/test_data/vat_flux.go @@ -0,0 +1,52 @@ +/* + * Copyright 2018 Vulcanize + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test_data + +import ( + "encoding/json" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +var VatFluxLog = types.Log{ + Address: common.HexToAddress("0x239e6f0ab02713f1f8aa90ebeded9fc66dc96cd6"), + Topics: []common.Hash{ + common.HexToHash("0xa6e4182100000000000000000000000000000000000000000000000000000000"), + common.HexToHash("0x0000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca"), + common.HexToHash("0x0000000000000000000000007fa9ef6609ca7921112231f8f195138ebba29770"), + common.HexToHash("0x00000000000000000000000093086347c52a8878af71bb818509d484c6a2e1bf"), + }, + Data: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000084a6e418217340e006f4135ba6970d43bf43d88dcad4e7a8ca00000000000000000000000007fa9ef6609ca7921112231f8f195138ebba297700000000000000000000000093086347c52a8878af71bb818509d484c6a2e1bf000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"), + BlockNumber: 23, + TxHash: common.HexToHash("0xf98681bab9b8c75bd8aa4a7d0a8142ff527c5ea8fa54f3c2835d4533838b2e6f"), + TxIndex: 0, + BlockHash: common.HexToHash("0xc3fe212ad4f81ade1265af6de2b4bb50d962b1a4db06aabc982e7f9cb0972c2d"), + Index: 0, + Removed: false, +} + +var rawFluxLog, _ = json.Marshal(VatFluxLog) +var VatFluxModel = vat_flux.VatFluxModel{ + Ilk: "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA", + Src: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977", + Dst: "0x93086347c52a8878af71bB818509d484c6a2e1bF", + Rad: "123", + TransactionIndex: 0, + Raw: rawFluxLog, +} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index e13e6c59..a7e5bb8c 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -37,6 +37,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/price_feeds" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_grab" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_heal" @@ -75,6 +76,7 @@ var ( VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer + VatFluxTransformerInitializer = vat_flux.VatFluxTransformerInitializer{Config: vat_flux.VatFluxConfig}.NewVatFluxTransformer ) func TransformerInitializers() []shared.TransformerInitializer { @@ -104,5 +106,6 @@ func TransformerInitializers() []shared.TransformerInitializer { VatFoldTransformerInitializer, VatTollTransformerInitializer, VatTuneTransformerInitializer, + VatFluxTransformerInitializer, } } diff --git a/pkg/transformers/vat_flux/config.go b/pkg/transformers/vat_flux/config.go new file mode 100644 index 00000000..c7f3449f --- /dev/null +++ b/pkg/transformers/vat_flux/config.go @@ -0,0 +1,11 @@ +package vat_flux + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var VatFluxConfig = shared.TransformerConfig{ + ContractAddresses: []string{shared.VatContractAddress}, + ContractAbi: shared.VatABI, + Topics: []string{shared.VatFluxSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/vat_flux/converter.go b/pkg/transformers/vat_flux/converter.go new file mode 100644 index 00000000..47dac29a --- /dev/null +++ b/pkg/transformers/vat_flux/converter.go @@ -0,0 +1,75 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +import ( + "encoding/json" + "errors" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "math/big" +) + +type Converter interface { + ToModels(ethLogs []types.Log) ([]VatFluxModel, error) +} + +type VatFluxConverter struct{} + +func (VatFluxConverter) ToModels(ethLogs []types.Log) ([]VatFluxModel, error) { + var models []VatFluxModel + for _, ethLog := range ethLogs { + err := verifyLog(ethLog) + if err != nil { + return nil, err + } + + ilk := common.BytesToAddress(ethLog.Topics[1].Bytes()[:]) + src := common.BytesToAddress(ethLog.Topics[2].Bytes()[:]) + dst := common.BytesToAddress(ethLog.Topics[3].Bytes()[:]) + radBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) + rad := big.NewInt(0).SetBytes(radBytes).String() + + if err != nil { + return nil, err + } + + rawLogJson, err := json.Marshal(ethLog) + if err != nil { + return nil, err + } + + model := VatFluxModel{ + Ilk: ilk.String(), + Src: src.String(), + Dst: dst.String(), + Rad: rad, + TransactionIndex: ethLog.TxIndex, + Raw: rawLogJson, + } + + models = append(models, model) + } + + return models, nil +} + +func verifyLog(log types.Log) error { + if len(log.Topics) < 4 { + return errors.New("log missing topics") + } + return nil +} diff --git a/pkg/transformers/vat_flux/converter_test.go b/pkg/transformers/vat_flux/converter_test.go new file mode 100644 index 00000000..d84ac555 --- /dev/null +++ b/pkg/transformers/vat_flux/converter_test.go @@ -0,0 +1,54 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +var _ = Describe("VatFlux converter", func() { + It("Converts logs to models", func() { + converter := vat_flux.VatFluxConverter{} + models, err := converter.ToModels([]types.Log{test_data.VatFluxLog}) + + Expect(err).NotTo(HaveOccurred()) + Expect(models[0].Ilk).To(Equal(test_data.VatFluxModel.Ilk)) + Expect(models[0].Src).To(Equal(test_data.VatFluxModel.Src)) + Expect(models[0].Dst).To(Equal(test_data.VatFluxModel.Dst)) + Expect(models[0].Rad).To(Equal(test_data.VatFluxModel.Rad)) + Expect(models[0].TransactionIndex).To(Equal(test_data.VatFluxModel.TransactionIndex)) + Expect(models[0].Raw).To(Equal(test_data.VatFluxModel.Raw)) + }) + + It("Returns an error there are missing topics", func() { + converter := vat_flux.VatFluxConverter{} + badLog := types.Log{ + Topics: []common.Hash{ + common.HexToHash("0x"), + common.HexToHash("0x"), + common.HexToHash("0x"), + }, + } + _, err := converter.ToModels([]types.Log{badLog}) + + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/transformers/vat_flux/model.go b/pkg/transformers/vat_flux/model.go new file mode 100644 index 00000000..a63310af --- /dev/null +++ b/pkg/transformers/vat_flux/model.go @@ -0,0 +1,24 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +type VatFluxModel struct { + Ilk string + Src string + Dst string + Rad string + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go new file mode 100644 index 00000000..412629b6 --- /dev/null +++ b/pkg/transformers/vat_flux/repository.go @@ -0,0 +1,84 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type Repository interface { + Create(headerId int64, models []VatFluxModel) error + MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) + MarkCheckedHeader(headerId int64) error +} + +type VatFluxRepository struct { + DB *postgres.DB +} + +func NewVatFluxRepository(db *postgres.DB) VatFluxRepository { + return VatFluxRepository{DB: db} +} + +func (repository VatFluxRepository) Create(headerId int64, models []VatFluxModel) error { + tx, err := repository.DB.Begin() + if err != nil { + return err + } + + for _, model := range models { + _, err := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, raw_log) + VALUES($1, $2, $3, $4, $5, $6, $7)`, + headerId, model.Ilk, model.Dst, model.Src, model.Rad, model.TransactionIndex, model.Raw) + if err != nil { + tx.Rollback() + return err + } + } + + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_flux_checked) + VALUES($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_flux_checked = $2`, headerId, true) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func (repository VatFluxRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { + var headers []core.Header + err := repository.DB.Select(&headers, + `SELECT headers.id, block_number from headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR vat_flux_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, + startingBlock, endingBlock, repository.DB.Node.ID) + + return headers, err +} + +func (repository VatFluxRepository) MarkCheckedHeader(headerId int64) error { + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, vat_flux_checked) + VALUES($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET vat_flux_checked = $2`, headerId, true) + + return err +} diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go new file mode 100644 index 00000000..b472e84b --- /dev/null +++ b/pkg/transformers/vat_flux/repository_test.go @@ -0,0 +1,226 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vat_flux_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("VatFlux Repository", func() { + var db *postgres.DB + var repository vat_flux.VatFluxRepository + var headerRepository repositories.HeaderRepository + var headerId int64 + var err error + + BeforeEach(func() { + node := test_config.NewTestNode() + db = test_config.NewTestDB(node) + test_config.CleanTestDB(db) + repository = vat_flux.VatFluxRepository{DB: db} + headerRepository = repositories.NewHeaderRepository(db) + headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + }) + + type VatFluxDBResult struct { + vat_flux.VatFluxModel + Id int + HeaderId int64 `db:"header_id"` + } + + type CheckedHeaderResult struct { + VatFluxChecked bool `db:"vat_flux_checked"` + } + + Describe("Create", func() { + It("persists vat flux records", func() { + anotherVatFlux := test_data.VatFluxModel + anotherVatFlux.TransactionIndex = test_data.VatFluxModel.TransactionIndex + 1 + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel, anotherVatFlux}) + + var dbResult []VatFluxDBResult + err = db.Select(&dbResult, `SELECT * from maker.vat_flux where header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(len(dbResult)).To(Equal(2)) + Expect(dbResult[0].Ilk).To(Equal(test_data.VatFluxModel.Ilk)) + Expect(dbResult[0].Dst).To(Equal(test_data.VatFluxModel.Dst)) + Expect(dbResult[0].Src).To(Equal(test_data.VatFluxModel.Src)) + Expect(dbResult[0].Rad).To(Equal(test_data.VatFluxModel.Rad)) + Expect(dbResult[0].TransactionIndex).To(Equal(test_data.VatFluxModel.TransactionIndex)) + Expect(dbResult[1].TransactionIndex).To(Equal(test_data.VatFluxModel.TransactionIndex + 1)) + Expect(dbResult[0].Raw).To(MatchJSON(test_data.VatFluxModel.Raw)) + Expect(dbResult[0].HeaderId).To(Equal(headerId)) + }) + + It("returns an error if the insertion fails", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).NotTo(HaveOccurred()) + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) + }) + + It("marks the header as checked for vat flux logs", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).NotTo(HaveOccurred()) + + var headerChecked bool + err = db.Get(&headerChecked, `SELECT vat_flux_checked FROM public.checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(headerChecked).To(BeTrue()) + }) + + It("removes vat flux if corresponding header is deleted", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel}) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Exec(`DELETE FROM headers WHERE id = $1`, headerId) + + Expect(err).NotTo(HaveOccurred()) + var count int + err = db.QueryRow(`SELECT count(*) from maker.vat_flux`).Scan(&count) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(0)) + }) + + It("wraps create in a transaction", func() { + err = repository.Create(headerId, []vat_flux.VatFluxModel{test_data.VatFluxModel, test_data.VatFluxModel}) + Expect(err).To(HaveOccurred()) + var count int + err = repository.DB.QueryRowx(`SELECT count(*) FROM maker.vat_flux`).Scan(&count) + Expect(count).To(Equal(0)) + }) + }) + + Describe("MissingHeaders", func() { + It("returns headers that haven't been checked", func() { + startingBlock := GinkgoRandomSeed() + vatFluxBlock := startingBlock + 1 + endingBlock := startingBlock + 2 + outsideRangeBlock := startingBlock + 3 + + var headerIds []int64 + blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} + for _, n := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + } + + err = repository.MarkCheckedHeader(headerIds[0]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := repository.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(headers[0].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(headers[1].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(len(headers)).To(Equal(2)) + }) + + It("returns header ids when checked_headers.vat_flux is false", func() { + startingBlock := GinkgoRandomSeed() + vatFluxBlock := startingBlock + 1 + endingBlock := startingBlock + 2 + outsideRangeBlock := startingBlock + 3 + + var headerIds []int64 + blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} + for _, n := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + } + + err = repository.MarkCheckedHeader(headerIds[0]) + _, err = repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES ($1)`, headerIds[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := repository.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(headers[0].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(headers[1].Id).To(Or(Equal(headerIds[1]), Equal(headerIds[2]))) + Expect(len(headers)).To(Equal(2)) + }) + + It("only returns header ids for the current node", func() { + startingBlock := GinkgoRandomSeed() + vatFluxBlock := startingBlock + 1 + endingBlock := startingBlock + 2 + outsideRangeBlock := startingBlock + 3 + db2 := test_config.NewTestDB(core.Node{ID: "second node"}) + headerRepository2 := repositories.NewHeaderRepository(db2) + repository2 := vat_flux.NewVatFluxRepository(db2) + + var headerIds []int64 + blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} + for _, n := range blockNumbers { + headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + headerIds = append(headerIds, headerId) + + _, err = headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + Expect(err).NotTo(HaveOccurred()) + } + + err = repository.MarkCheckedHeader(headerIds[0]) + Expect(err).NotTo(HaveOccurred()) + + nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(2)) + + nodeTwoMissingHeaders, err := repository2.MissingHeaders(startingBlock, endingBlock) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) + }) + }) + + Describe("MarkCheckedHeader", func() { + It("creates a new checked_header record", func() { + err := repository.MarkCheckedHeader(headerId) + Expect(err).NotTo(HaveOccurred()) + + var checkedHeaderResult = CheckedHeaderResult{} + err = db.Get(&checkedHeaderResult, `SELECT vat_flux_checked FROM checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFluxChecked).To(BeTrue()) + }) + + It("updates an existing checked header", func() { + _, err := repository.DB.Exec(`INSERT INTO checked_headers (header_id) VALUES($1)`, headerId) + Expect(err).NotTo(HaveOccurred()) + + var checkedHeaderResult CheckedHeaderResult + err = db.Get(&checkedHeaderResult, `SELECT vat_flux_checked FROM checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFluxChecked).To(BeFalse()) + + err = repository.MarkCheckedHeader(headerId) + Expect(err).NotTo(HaveOccurred()) + + err = db.Get(&checkedHeaderResult, `SELECT vat_flux_checked FROM checked_headers WHERE header_id = $1`, headerId) + Expect(err).NotTo(HaveOccurred()) + Expect(checkedHeaderResult.VatFluxChecked).To(BeTrue()) + }) + }) +}) diff --git a/pkg/transformers/vat_flux/transformer.go b/pkg/transformers/vat_flux/transformer.go new file mode 100644 index 00000000..0d0d8bf8 --- /dev/null +++ b/pkg/transformers/vat_flux/transformer.go @@ -0,0 +1,63 @@ +package vat_flux + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "log" +) + +type VatFluxTransformerInitializer struct { + Config shared.TransformerConfig +} + +func (initializer VatFluxTransformerInitializer) NewVatFluxTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + converter := VatFluxConverter{} + fetcher := shared.NewFetcher(blockChain) + repository := NewVatFluxRepository(db) + return VatFluxTransformer{ + Config: initializer.Config, + Converter: converter, + Fetcher: fetcher, + Repository: repository, + } +} + +type VatFluxTransformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer VatFluxTransformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + log.Printf("Fetching vat flux event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatFluxSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(VatFluxConfig.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + if len(matchingLogs) < 1 { + err = transformer.Repository.MarkCheckedHeader(header.Id) + if err != nil { + return err + } + continue + } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/transformers/vat_flux/transformer_test.go b/pkg/transformers/vat_flux/transformer_test.go new file mode 100644 index 00000000..18f68e27 --- /dev/null +++ b/pkg/transformers/vat_flux/transformer_test.go @@ -0,0 +1,178 @@ +package vat_flux_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + vat_flux_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_flux" + "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" +) + +var _ = Describe("Vat flux transformer", func() { + It("gets missing headers for block numbers specified in config", func() { + repository := &vat_flux_mocks.MockVatFluxRepository{} + transformer := vat_flux.VatFluxTransformer{ + Config: vat_flux.VatFluxConfig, + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedStartingBlockNumber).To(Equal(vat_flux.VatFluxConfig.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(vat_flux.VatFluxConfig.EndingBlockNumber)) + }) + + It("returns error if repository returns error for missing headers", func() { + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeadersErr(fakes.FakeError) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: &mocks.MockLogFetcher{}, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("fetches logs for missing headers", func() { + fetcher := &mocks.MockLogFetcher{} + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_flux.VatFluxConfig.ContractAddresses, vat_flux.VatFluxConfig.ContractAddresses})) + Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatFluxSignature)}})) + }) + + It("returns error if fetcher returns error", func() { + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetcherError(fakes.FakeError) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: &vat_flux_mocks.MockVatFlux{}, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("returns error if marking header checked returns err", func() { + mockConverter := &vat_flux_mocks.MockVatFlux{} + mockRepository := &vat_flux_mocks.MockVatFluxRepository{} + mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} + transformer := vat_flux.VatFluxTransformer{ + Converter: mockConverter, + Fetcher: mockFetcher, + Repository: mockRepository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("converts matching logs", func() { + converter := &vat_flux_mocks.MockVatFlux{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.VatFluxLog})) + }) + + It("returns error if converter returns error", func() { + converter := &vat_flux_mocks.MockVatFlux{} + converter.SetConverterError(fakes.FakeError) + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("persists vat flux model", func() { + converter := &vat_flux_mocks.MockVatFlux{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + fakeHeader := core.Header{BlockNumber: 1, Id: 2} + repository.SetMissingHeaders([]core.Header{fakeHeader}) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) + Expect(repository.PassedModels).To(Equal([]vat_flux.VatFluxModel{test_data.VatFluxModel})) + }) + + It("returns error if repository returns error for create", func() { + converter := &vat_flux_mocks.MockVatFlux{} + fetcher := &mocks.MockLogFetcher{} + fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) + repository := &vat_flux_mocks.MockVatFluxRepository{} + repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetCreateError(fakes.FakeError) + transformer := vat_flux.VatFluxTransformer{ + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) +}) diff --git a/pkg/transformers/vat_flux/vat_flux_suite_test.go b/pkg/transformers/vat_flux/vat_flux_suite_test.go new file mode 100644 index 00000000..9a099bf2 --- /dev/null +++ b/pkg/transformers/vat_flux/vat_flux_suite_test.go @@ -0,0 +1,19 @@ +package vat_flux_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestVatFlux(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VatFlux Suite") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/transformers/vat_grab/tranformer.go b/pkg/transformers/vat_grab/tranformer.go index 47dc2820..6dd40aaf 100644 --- a/pkg/transformers/vat_grab/tranformer.go +++ b/pkg/transformers/vat_grab/tranformer.go @@ -36,7 +36,7 @@ func (transformer VatGrabTransformer) Execute() error { if err != nil { return err } - log.Printf("Fetching vat init event logs for %d headers \n", len(missingHeaders)) + log.Printf("Fetching vat grab event logs for %d headers \n", len(missingHeaders)) for _, header := range missingHeaders { topics := [][]common.Hash{{common.HexToHash(shared.VatGrabSignature)}} matchingLogs, err := transformer.Fetcher.FetchLogs(VatGrabConfig.ContractAddresses, topics, header.BlockNumber) diff --git a/test_config/test_config.go b/test_config/test_config.go index a0c03d9e..8d0403bd 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -103,6 +103,7 @@ func CleanTestDB(db *postgres.DB) { db.MustExec("DELETE FROM maker.vat_fold") db.MustExec("DELETE FROM maker.vat_toll") db.MustExec("DELETE FROM maker.vat_tune") + db.MustExec("DELETE FROM maker.vat_flux") db.MustExec("DELETE FROM receipts") db.MustExec("DELETE FROM transactions") db.MustExec("DELETE FROM watched_contracts") From f32a4fa2a5f1dc4636ecc22edf0deaeb538e0a8a Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Wed, 17 Oct 2018 10:55:30 -0500 Subject: [PATCH 3/6] use kovan vat contract address --- pkg/transformers/shared/constants.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index df43dc7c..eec31b22 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -34,8 +34,7 @@ var ( PipContractAddress = "0x9FfFE440258B79c5d6604001674A4722FfC0f7Bc" PitContractAddress = "0xe7cf3198787c9a4daac73371a38f29aaeeced87e" RepContractAddress = "0xf88bbdc1e2718f8857f30a180076ec38d53cf296" - //VatContractAddress = "0xcd726790550afcd77e9a7a47e86a3f9010af126b" - VatContractAddress = "0x239e6f0ab02713f1f8aa90ebeded9fc66dc96cd6" + VatContractAddress = "0xcd726790550afcd77e9a7a47e86a3f9010af126b" //TODO: get pit and drip file method signatures directly from the ABI biteMethod = GetSolidityMethodSignature(CatABI, "Bite") From 2f64269ab39811789a6140ffb8c14eee7eb2ee12 Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Wed, 17 Oct 2018 11:48:57 -0500 Subject: [PATCH 4/6] update from pr feedback; update tests and use shared fake header --- pkg/transformers/test_data/vat_flux.go | 6 +- pkg/transformers/vat_flux/converter_test.go | 1 + pkg/transformers/vat_flux/repository.go | 2 +- pkg/transformers/vat_flux/repository_test.go | 11 ++-- pkg/transformers/vat_flux/transformer_test.go | 61 +++++++++++++++++++ 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/pkg/transformers/test_data/vat_flux.go b/pkg/transformers/test_data/vat_flux.go index 76dc4f49..5eb98bc5 100644 --- a/pkg/transformers/test_data/vat_flux.go +++ b/pkg/transformers/test_data/vat_flux.go @@ -28,8 +28,8 @@ var VatFluxLog = types.Log{ Address: common.HexToAddress("0x239e6f0ab02713f1f8aa90ebeded9fc66dc96cd6"), Topics: []common.Hash{ common.HexToHash("0xa6e4182100000000000000000000000000000000000000000000000000000000"), - common.HexToHash("0x0000000000000000000000007340e006f4135ba6970d43bf43d88dcad4e7a8ca"), - common.HexToHash("0x0000000000000000000000007fa9ef6609ca7921112231f8f195138ebba29770"), + common.HexToHash("0x0000000000000000000000007340e006f4135BA6970D43bf43d88DCAD4e7a8CA"), + common.HexToHash("0x0000000000000000000000007FA9EF6609Ca7921112231f8f195138ebba29770"), common.HexToHash("0x00000000000000000000000093086347c52a8878af71bb818509d484c6a2e1bf"), }, Data: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000084a6e418217340e006f4135ba6970d43bf43d88dcad4e7a8ca00000000000000000000000007fa9ef6609ca7921112231f8f195138ebba297700000000000000000000000093086347c52a8878af71bb818509d484c6a2e1bf000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b"), @@ -44,7 +44,7 @@ var VatFluxLog = types.Log{ var rawFluxLog, _ = json.Marshal(VatFluxLog) var VatFluxModel = vat_flux.VatFluxModel{ Ilk: "0x7340e006f4135BA6970D43bf43d88DCAD4e7a8CA", - Src: "0x07Fa9eF6609cA7921112231F8f195138ebbA2977", + Src: "0x7FA9EF6609Ca7921112231f8f195138ebba29770", Dst: "0x93086347c52a8878af71bB818509d484c6a2e1bF", Rad: "123", TransactionIndex: 0, diff --git a/pkg/transformers/vat_flux/converter_test.go b/pkg/transformers/vat_flux/converter_test.go index d84ac555..56bd4f38 100644 --- a/pkg/transformers/vat_flux/converter_test.go +++ b/pkg/transformers/vat_flux/converter_test.go @@ -30,6 +30,7 @@ var _ = Describe("VatFlux converter", func() { models, err := converter.ToModels([]types.Log{test_data.VatFluxLog}) Expect(err).NotTo(HaveOccurred()) + Expect(len(models)).To(Equal(1)) Expect(models[0].Ilk).To(Equal(test_data.VatFluxModel.Ilk)) Expect(models[0].Src).To(Equal(test_data.VatFluxModel.Src)) Expect(models[0].Dst).To(Equal(test_data.VatFluxModel.Dst)) diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index 412629b6..3521341f 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -41,7 +41,7 @@ func (repository VatFluxRepository) Create(headerId int64, models []VatFluxModel for _, model := range models { _, err := tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6, $7)`, + VALUES($1, $2, $3, $4, $5::numeric, $6, $7)`, headerId, model.Ilk, model.Dst, model.Src, model.Rad, model.TransactionIndex, model.Raw) if err != nil { tx.Rollback() diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go index b472e84b..140917d5 100644 --- a/pkg/transformers/vat_flux/repository_test.go +++ b/pkg/transformers/vat_flux/repository_test.go @@ -20,6 +20,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" "github.com/vulcanize/vulcanizedb/test_config" @@ -38,7 +39,7 @@ var _ = Describe("VatFlux Repository", func() { test_config.CleanTestDB(db) repository = vat_flux.VatFluxRepository{DB: db} headerRepository = repositories.NewHeaderRepository(db) - headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) }) @@ -122,7 +123,7 @@ var _ = Describe("VatFlux Repository", func() { var headerIds []int64 blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} for _, n := range blockNumbers { - headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerId, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) headerIds = append(headerIds, headerId) } @@ -146,7 +147,7 @@ var _ = Describe("VatFlux Repository", func() { var headerIds []int64 blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} for _, n := range blockNumbers { - headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerId, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) headerIds = append(headerIds, headerId) } @@ -174,11 +175,11 @@ var _ = Describe("VatFlux Repository", func() { var headerIds []int64 blockNumbers := []int64{startingBlock, vatFluxBlock, endingBlock, outsideRangeBlock} for _, n := range blockNumbers { - headerId, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerId, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) headerIds = append(headerIds, headerId) - _, err = headerRepository2.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + _, err = headerRepository2.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } diff --git a/pkg/transformers/vat_flux/transformer_test.go b/pkg/transformers/vat_flux/transformer_test.go index 18f68e27..e23da618 100644 --- a/pkg/transformers/vat_flux/transformer_test.go +++ b/pkg/transformers/vat_flux/transformer_test.go @@ -15,6 +15,55 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_flux" ) +type setupOptions struct { + setMissingHeadersError bool + setFetcherError bool + setConverterError bool + setCreateError bool + fetchedLogs []types.Log + missingHeaders []core.Header +} + +func setup(options setupOptions) ( + vat_flux.VatFluxTransformer, + *mocks.MockLogFetcher, + *vat_flux_mocks.MockVatFlux, + *vat_flux_mocks.MockVatFluxRepository, +) { + fetcher := &mocks.MockLogFetcher{} + if options.setFetcherError { + fetcher.SetFetcherError(fakes.FakeError) + } + if len(options.fetchedLogs) > 0 { + fetcher.SetFetchedLogs(options.fetchedLogs) + } + + converter := &vat_flux_mocks.MockVatFlux{} + if options.setConverterError { + converter.SetConverterError(fakes.FakeError) + } + + repository := &vat_flux_mocks.MockVatFluxRepository{} + if options.setMissingHeadersError { + repository.SetMissingHeadersErr(fakes.FakeError) + } + if options.setCreateError { + repository.SetCreateError(fakes.FakeError) + } + if len(options.missingHeaders) > 0 { + repository.SetMissingHeaders(options.missingHeaders) + } + + transformer := vat_flux.VatFluxTransformer{ + Config: vat_flux.VatFluxConfig, + Fetcher: fetcher, + Converter: converter, + Repository: repository, + } + + return transformer, fetcher, converter, repository +} + var _ = Describe("Vat flux transformer", func() { It("gets missing headers for block numbers specified in config", func() { repository := &vat_flux_mocks.MockVatFluxRepository{} @@ -47,6 +96,18 @@ var _ = Describe("Vat flux transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) + It("marks the header as checked when there are no logs", func() { + header := core.Header{Id: GinkgoRandomSeed()} + transformer, _, _, repository := setup(setupOptions{ + missingHeaders: []core.Header{header}, + }) + + err := transformer.Execute() + + Expect(err).NotTo(HaveOccurred()) + Expect(repository.MarkHeaderCheckedPassedHeaderID).To(Equal(header.Id)) + }) + It("fetches logs for missing headers", func() { fetcher := &mocks.MockLogFetcher{} repository := &vat_flux_mocks.MockVatFluxRepository{} From a3ca319e9b28e30f141d12ee6e2a7b78fabdd117 Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Wed, 17 Oct 2018 12:04:11 -0500 Subject: [PATCH 5/6] remove unnecessary byte slicing --- pkg/transformers/vat_flux/converter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/transformers/vat_flux/converter.go b/pkg/transformers/vat_flux/converter.go index 47dac29a..817a00d3 100644 --- a/pkg/transformers/vat_flux/converter.go +++ b/pkg/transformers/vat_flux/converter.go @@ -37,9 +37,9 @@ func (VatFluxConverter) ToModels(ethLogs []types.Log) ([]VatFluxModel, error) { return nil, err } - ilk := common.BytesToAddress(ethLog.Topics[1].Bytes()[:]) - src := common.BytesToAddress(ethLog.Topics[2].Bytes()[:]) - dst := common.BytesToAddress(ethLog.Topics[3].Bytes()[:]) + ilk := common.BytesToAddress(ethLog.Topics[1].Bytes()) + src := common.BytesToAddress(ethLog.Topics[2].Bytes()) + dst := common.BytesToAddress(ethLog.Topics[3].Bytes()) radBytes := shared.GetDataBytesAtIndex(-1, ethLog.Data) rad := big.NewInt(0).SetBytes(radBytes).String() From 4721a91faae17bb20d9226e0738d344fd81988c9 Mon Sep 17 00:00:00 2001 From: Taka Goto Date: Wed, 17 Oct 2018 12:15:15 -0500 Subject: [PATCH 6/6] use before each for transformer test --- pkg/transformers/vat_flux/transformer_test.go | 146 ++++-------------- 1 file changed, 33 insertions(+), 113 deletions(-) diff --git a/pkg/transformers/vat_flux/transformer_test.go b/pkg/transformers/vat_flux/transformer_test.go index e23da618..d621e34c 100644 --- a/pkg/transformers/vat_flux/transformer_test.go +++ b/pkg/transformers/vat_flux/transformer_test.go @@ -24,56 +24,29 @@ type setupOptions struct { missingHeaders []core.Header } -func setup(options setupOptions) ( - vat_flux.VatFluxTransformer, - *mocks.MockLogFetcher, - *vat_flux_mocks.MockVatFlux, - *vat_flux_mocks.MockVatFluxRepository, -) { - fetcher := &mocks.MockLogFetcher{} - if options.setFetcherError { - fetcher.SetFetcherError(fakes.FakeError) - } - if len(options.fetchedLogs) > 0 { - fetcher.SetFetchedLogs(options.fetchedLogs) - } - - converter := &vat_flux_mocks.MockVatFlux{} - if options.setConverterError { - converter.SetConverterError(fakes.FakeError) - } - - repository := &vat_flux_mocks.MockVatFluxRepository{} - if options.setMissingHeadersError { - repository.SetMissingHeadersErr(fakes.FakeError) - } - if options.setCreateError { - repository.SetCreateError(fakes.FakeError) - } - if len(options.missingHeaders) > 0 { - repository.SetMissingHeaders(options.missingHeaders) - } - - transformer := vat_flux.VatFluxTransformer{ - Config: vat_flux.VatFluxConfig, - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } - - return transformer, fetcher, converter, repository -} - var _ = Describe("Vat flux transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &vat_flux_mocks.MockVatFluxRepository{} - transformer := vat_flux.VatFluxTransformer{ - Config: vat_flux.VatFluxConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &vat_flux_mocks.MockVatFlux{}, + var ( + config shared.TransformerConfig + converter *vat_flux_mocks.MockVatFlux + fetcher *mocks.MockLogFetcher + repository *vat_flux_mocks.MockVatFluxRepository + transformer vat_flux.VatFluxTransformer + ) + + BeforeEach(func() { + config = vat_flux.VatFluxConfig + converter = &vat_flux_mocks.MockVatFlux{} + fetcher = &mocks.MockLogFetcher{} + repository = &vat_flux_mocks.MockVatFluxRepository{} + transformer = vat_flux.VatFluxTransformer{ + Config: config, + Converter: converter, + Fetcher: fetcher, Repository: repository, } + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -82,14 +55,7 @@ var _ = Describe("Vat flux transformer", func() { }) It("returns error if repository returns error for missing headers", func() { - repository := &vat_flux_mocks.MockVatFluxRepository{} repository.SetMissingHeadersErr(fakes.FakeError) - transformer := vat_flux.VatFluxTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &vat_flux_mocks.MockVatFlux{}, - Repository: repository, - } - err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -98,10 +64,7 @@ var _ = Describe("Vat flux transformer", func() { It("marks the header as checked when there are no logs", func() { header := core.Header{Id: GinkgoRandomSeed()} - transformer, _, _, repository := setup(setupOptions{ - missingHeaders: []core.Header{header}, - }) - + repository.SetMissingHeaders([]core.Header{header}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -109,14 +72,7 @@ var _ = Describe("Vat flux transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &vat_flux_mocks.MockVatFluxRepository{} repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := vat_flux.VatFluxTransformer{ - Fetcher: fetcher, - Converter: &vat_flux_mocks.MockVatFlux{}, - Repository: repository, - } err := transformer.Execute() @@ -127,13 +83,22 @@ var _ = Describe("Vat flux transformer", func() { }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &vat_flux_mocks.MockVatFluxRepository{} repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + + err := transformer.Execute() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + + It("returns error if marking header checked returns err", func() { + repository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) + repository.SetMarkHeaderCheckedErr(fakes.FakeError) + mockFetcher := &mocks.MockLogFetcher{} transformer := vat_flux.VatFluxTransformer{ - Fetcher: fetcher, - Converter: &vat_flux_mocks.MockVatFlux{}, + Converter: converter, + Fetcher: mockFetcher, Repository: repository, } @@ -143,29 +108,8 @@ var _ = Describe("Vat flux transformer", func() { Expect(err).To(MatchError(fakes.FakeError)) }) - It("returns error if marking header checked returns err", func() { - mockConverter := &vat_flux_mocks.MockVatFlux{} - mockRepository := &vat_flux_mocks.MockVatFluxRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := vat_flux.VatFluxTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } - - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - It("converts matching logs", func() { - converter := &vat_flux_mocks.MockVatFlux{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) - repository := &vat_flux_mocks.MockVatFluxRepository{} repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) transformer := vat_flux.VatFluxTransformer{ Fetcher: fetcher, @@ -180,17 +124,9 @@ var _ = Describe("Vat flux transformer", func() { }) It("returns error if converter returns error", func() { - converter := &vat_flux_mocks.MockVatFlux{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) - repository := &vat_flux_mocks.MockVatFluxRepository{} repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vat_flux.VatFluxTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() @@ -199,17 +135,9 @@ var _ = Describe("Vat flux transformer", func() { }) It("persists vat flux model", func() { - converter := &vat_flux_mocks.MockVatFlux{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) - repository := &vat_flux_mocks.MockVatFluxRepository{} fakeHeader := core.Header{BlockNumber: 1, Id: 2} repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := vat_flux.VatFluxTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() @@ -219,17 +147,9 @@ var _ = Describe("Vat flux transformer", func() { }) It("returns error if repository returns error for create", func() { - converter := &vat_flux_mocks.MockVatFlux{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.VatFluxLog}) - repository := &vat_flux_mocks.MockVatFluxRepository{} repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) repository.SetCreateError(fakes.FakeError) - transformer := vat_flux.VatFluxTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute()