From 1b9e6491da2b112dc83b375ce564aee1ed536742 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 5 Oct 2018 13:41:38 +0200 Subject: [PATCH 01/36] Add transformer factory --- pkg/transformers/factories/transformer.go | 101 ++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 pkg/transformers/factories/transformer.go diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go new file mode 100644 index 00000000..dd50edd9 --- /dev/null +++ b/pkg/transformers/factories/transformer.go @@ -0,0 +1,101 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package factories + +import ( + "github.com/ethereum/go-ethereum/core/types" + "log" + + "github.com/ethereum/go-ethereum/common" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type TransformerInitializer struct { + Config shared.TransformerConfig + Converter Converter + Repository Repository +} + +type Model struct { + TransactionIndex uint `db:"tx_idx"` + Raw []byte `db:"raw_log"` +} + +type Converter interface { + ToModel(ethLog types.Log) (Model, error) +} + +type Repository interface { + Create(headerID int64, models []Model) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) +} + + +func (initializer TransformerInitializer) NewTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { + fetcher := shared.NewFetcher(blockChain) + return Transformer{ + Config: initializer.Config, + Converter: initializer.Converter, + Fetcher: fetcher, + Repository: initializer.Repository, + } +} + +type Transformer struct { + Config shared.TransformerConfig + Converter Converter + Fetcher shared.LogFetcher + Repository Repository +} + +func (transformer Transformer) Execute() error { + missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + if err != nil { + return err + } + + log.Printf("Fetching vat move event logs for %d headers \n", len(missingHeaders)) + for _, header := range missingHeaders { + topics := [][]common.Hash{{common.HexToHash(shared.VatMoveSignature)}} + matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topics, header.BlockNumber) + if err != nil { + return err + } + + if len(matchingLogs) < 1 { + err := transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + return err + } + } + + for _, log := range matchingLogs { + model, err := transformer.Converter.ToModel(log) + if err != nil { + return err + } + + err = transformer.Repository.Create(header.Id, []Model{model}) + if err != nil { + return err + } + } + } + return nil +} From 15e33db33d99ecee05b2286bca4c8451d834dc9d Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 9 Oct 2018 16:03:34 +0200 Subject: [PATCH 02/36] Fix transformer factory --- pkg/transformers/factories/transformer.go | 76 ++++++++----------- pkg/transformers/shared/log_fetcher.go | 9 +++ .../test_data/mocks/log_fetcher.go | 3 + 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index dd50edd9..a1d30b11 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -16,52 +16,37 @@ package factories import ( "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "log" "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -type TransformerInitializer struct { - Config shared.TransformerConfig - Converter Converter - Repository Repository -} - -type Model struct { - TransactionIndex uint `db:"tx_idx"` - Raw []byte `db:"raw_log"` -} - -type Converter interface { - ToModel(ethLog types.Log) (Model, error) -} - -type Repository interface { - Create(headerID int64, models []Model) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - - -func (initializer TransformerInitializer) NewTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - fetcher := shared.NewFetcher(blockChain) - return Transformer{ - Config: initializer.Config, - Converter: initializer.Converter, - Fetcher: fetcher, - Repository: initializer.Repository, - } -} - type Transformer struct { Config shared.TransformerConfig Converter Converter - Fetcher shared.LogFetcher Repository Repository + Fetcher shared.SettableLogFetcher +} + +type Converter interface { + ToModels(ethLog []types.Log) ([]interface{}, error) +} + +type Repository interface { + Create(headerID int64, models []interface{}) error + MarkHeaderChecked(headerID int64) error + MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) + SetDB(db *postgres.DB) +} + +func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { + transformer.Repository.SetDB(db) + transformer.Fetcher.SetBC(bc) + return transformer } func (transformer Transformer) Execute() error { @@ -72,6 +57,7 @@ func (transformer Transformer) Execute() error { log.Printf("Fetching vat move event logs for %d headers \n", len(missingHeaders)) for _, header := range missingHeaders { + // TODO Needs signature in config topics := [][]common.Hash{{common.HexToHash(shared.VatMoveSignature)}} matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topics, header.BlockNumber) if err != nil { @@ -85,16 +71,20 @@ func (transformer Transformer) Execute() error { } } - for _, log := range matchingLogs { - model, err := transformer.Converter.ToModel(log) - if err != nil { - return err - } + models, err := transformer.Converter.ToModels(matchingLogs) + if err != nil { + return err + } - err = transformer.Repository.Create(header.Id, []Model{model}) - if err != nil { - return err - } + // Can't assert the whole collection, need to wash types individually + var typelessModels []interface{} + for _, m := range models { + typelessModels = append(typelessModels, m.(interface{})) + } + + err = transformer.Repository.Create(header.Id, typelessModels) + if err != nil { + return err } } return nil diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index a42a18ff..46d4f3dc 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -28,10 +28,19 @@ type LogFetcher interface { FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) } +type SettableLogFetcher interface { + LogFetcher + SetBC(bc core.BlockChain) +} + type Fetcher struct { blockChain core.BlockChain } +func (fetcher *Fetcher) SetBC(bc core.BlockChain) { + fetcher.blockChain = bc +} + func NewFetcher(blockchain core.BlockChain) Fetcher { return Fetcher{ blockChain: blockchain, diff --git a/pkg/transformers/test_data/mocks/log_fetcher.go b/pkg/transformers/test_data/mocks/log_fetcher.go index 463bf878..b6b591a7 100644 --- a/pkg/transformers/test_data/mocks/log_fetcher.go +++ b/pkg/transformers/test_data/mocks/log_fetcher.go @@ -17,6 +17,7 @@ package mocks import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" ) type MockLogFetcher struct { @@ -35,6 +36,8 @@ func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]comm return mlf.FetchedLogs, mlf.fetcherError } +func (mlf *MockLogFetcher) SetBC(_ core.BlockChain) {} + func (mlf *MockLogFetcher) SetFetcherError(err error) { mlf.fetcherError = err } From af1dfbc4895070326603f082662756f5ff0a7d4a Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 9 Oct 2018 16:04:07 +0200 Subject: [PATCH 03/36] Implement vat_move transformer using factory --- .../test_data/mocks/vat_move/converter.go | 5 +- .../test_data/mocks/vat_move/repository.go | 10 ++- pkg/transformers/transformers.go | 8 +- pkg/transformers/vat_move/repository.go | 32 +++---- pkg/transformers/vat_move/transformer.go | 83 ------------------- 5 files changed, 29 insertions(+), 109 deletions(-) delete mode 100644 pkg/transformers/vat_move/transformer.go diff --git a/pkg/transformers/test_data/mocks/vat_move/converter.go b/pkg/transformers/test_data/mocks/vat_move/converter.go index b11d1830..6650e9ed 100644 --- a/pkg/transformers/test_data/mocks/vat_move/converter.go +++ b/pkg/transformers/test_data/mocks/vat_move/converter.go @@ -17,7 +17,6 @@ package vat_move import ( "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" - "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_move" ) type MockVatMoveConverter struct { @@ -25,9 +24,9 @@ type MockVatMoveConverter struct { PassedLogs []types.Log } -func (converter *MockVatMoveConverter) ToModels(ethLogs []types.Log) ([]vat_move.VatMoveModel, error) { +func (converter *MockVatMoveConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []vat_move.VatMoveModel{test_data.VatMoveModel}, converter.converterError + return []interface{}{test_data.VatMoveModel}, converter.converterError } func (converter *MockVatMoveConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/vat_move/repository.go b/pkg/transformers/test_data/mocks/vat_move/repository.go index 20d578e7..3b40753f 100644 --- a/pkg/transformers/test_data/mocks/vat_move/repository.go +++ b/pkg/transformers/test_data/mocks/vat_move/repository.go @@ -16,7 +16,7 @@ package vat_move import ( "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_move" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) type MockVatMoveRepository struct { @@ -26,12 +26,12 @@ type MockVatMoveRepository struct { PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []vat_move.VatMoveModel + PassedModels []interface{} CheckedHeaderIDs []int64 CheckedHeaderError error } -func (repository *MockVatMoveRepository) Create(headerID int64, models []vat_move.VatMoveModel) error { +func (repository *MockVatMoveRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models return repository.createError @@ -63,3 +63,7 @@ func (repository *MockVatMoveRepository) MarkHeaderChecked(headerId int64) error func (repository *MockVatMoveRepository) SetCheckedHeaderError(e error) { repository.CheckedHeaderError = e } + +func (repository *MockVatMoveRepository) SetDB(db *postgres.DB) { + // TODO Implement passed DB +} \ No newline at end of file diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 3e404697..313b7bdf 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -27,6 +27,7 @@ import ( ilk2 "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/flop_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" @@ -72,7 +73,12 @@ var ( TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer - VatMoveTransformerInitializer = vat_move.VatMoveTransformerInitializer{Config: vat_move.VatMoveConfig}.NewVatMoveTransformer + VatMoveTransformerInitializer = factories.Transformer{ + Config: vat_move.VatMoveConfig, + Converter: &vat_move.VatMoveConverter{}, + Repository: &vat_move.VatMoveRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer VatSlipTransformerInitializer = vat_slip.VatSlipTransformerInitializer{Config: vat_slip.VatSlipConfig}.NewVatSlipTransformer diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index 821c36d5..527936ab 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -19,29 +19,19 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []VatMoveModel) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) - MarkHeaderChecked(headerID int64) error -} - type VatMoveRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewVatMoveRepository(db *postgres.DB) VatMoveRepository { - return VatMoveRepository{ - db: db, - } -} - -func (repository VatMoveRepository) Create(headerID int64, models []VatMoveModel) error { - tx, err := repository.db.Begin() +func (repository VatMoveRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } - for _, vatMove := range models { + var vatMove VatMoveModel + for _, model := range models { + vatMove = model.(VatMoveModel) _, err = tx.Exec( `INSERT INTO maker.vat_move (header_id, src, dst, rad, tx_idx, raw_log) VALUES ($1, $2, $3, $4::NUMERIC, $5, $6)`, @@ -71,7 +61,7 @@ func (repository VatMoveRepository) Create(headerID int64, models []VatMoveModel func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -81,16 +71,20 @@ func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBl AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_move_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, vat_move_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET vat_move_checked = $2`, headerID, true) return err } + +func (repository *VatMoveRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/vat_move/transformer.go b/pkg/transformers/vat_move/transformer.go deleted file mode 100644 index 5318143c..00000000 --- a/pkg/transformers/vat_move/transformer.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package vat_move - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type VatMoveTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer VatMoveTransformerInitializer) NewVatMoveTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := VatMoveConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewVatMoveRepository(db) - return VatMoveTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -type VatMoveTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -func (transformer VatMoveTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - - log.Printf("Fetching vat move event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.VatMoveSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(VatMoveConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - - if len(matchingLogs) < 1 { - err := transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - continue - } - - models, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - - err = transformer.Repository.Create(header.Id, models) - if err != nil { - return err - } - } - return nil -} From 155569a1d8eafa55d8d2f725ce90e78e568d2d0c Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 9 Oct 2018 16:04:32 +0200 Subject: [PATCH 04/36] Update vat_move tests --- pkg/transformers/vat_move/repository_test.go | 11 ++-- pkg/transformers/vat_move/transformer_test.go | 50 +++++++++---------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index 6333f80b..978de862 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -38,7 +38,7 @@ var _ = Describe("Vat Move", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - vatMoveRepository = vat_move.NewVatMoveRepository(db) + vatMoveRepository = vat_move.VatMoveRepository{DB: db} }) Describe("Create", func() { @@ -48,7 +48,7 @@ var _ = Describe("Vat Move", func() { BeforeEach(func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = vatMoveRepository.Create(headerID, []vat_move.VatMoveModel{test_data.VatMoveModel}) + err = vatMoveRepository.Create(headerID, []interface{}{test_data.VatMoveModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -71,7 +71,7 @@ var _ = Describe("Vat Move", func() { }) It("returns an error if insertion fails", func() { - err = vatMoveRepository.Create(headerID, []vat_move.VatMoveModel{test_data.VatMoveModel}) + err = vatMoveRepository.Create(headerID, []interface{}{test_data.VatMoveModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) @@ -145,8 +145,8 @@ var _ = Describe("Vat Move", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - vatMoveRepositoryTwo := vat_move.NewVatMoveRepository(dbTwo) - err := vatMoveRepository.Create(headerIDs[0], []vat_move.VatMoveModel{test_data.VatMoveModel}) + vatMoveRepositoryTwo := vat_move.VatMoveRepository{DB: dbTwo} + err := vatMoveRepository.Create(headerIDs[0], []interface{}{test_data.VatMoveModel}) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := vatMoveRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) @@ -157,7 +157,6 @@ var _ = Describe("Vat Move", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) - }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/vat_move/transformer_test.go b/pkg/transformers/vat_move/transformer_test.go index 38371787..9b63cc06 100644 --- a/pkg/transformers/vat_move/transformer_test.go +++ b/pkg/transformers/vat_move/transformer_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" @@ -45,12 +46,12 @@ var _ = Describe("Vat move transformer", func() { }) It("gets missing headers for block numbers specified in config", func() { - transformer := vat_move.VatMoveTransformer{ - Config: config, + transformer := factories.Transformer{ + Config: vat_move.VatMoveConfig, Converter: &converter, - Fetcher: &fetcher, Repository: &repository, - } + Fetcher: &fetcher, + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -61,13 +62,12 @@ var _ = Describe("Vat move transformer", func() { It("returns error if repository returns error for missing headers", func() { repository.SetMissingHeadersError(fakes.FakeError) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Converter: &converter, Fetcher: &fetcher, Repository: &repository, - } - + }.NewTransformer(nil, nil) err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -76,12 +76,12 @@ var _ = Describe("Vat move transformer", func() { It("fetches logs for missing headers", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Fetcher: &fetcher, - Converter: &vat_move_mocks.MockVatMoveConverter{}, + Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -97,12 +97,12 @@ var _ = Describe("Vat move transformer", func() { It("returns error if fetcher returns error", func() { fetcher.SetFetcherError(fakes.FakeError) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -113,12 +113,12 @@ var _ = Describe("Vat move transformer", func() { It("converts matching logs", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -130,12 +130,12 @@ var _ = Describe("Vat move transformer", func() { converter.SetConverterError(fakes.FakeError) fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -146,12 +146,12 @@ var _ = Describe("Vat move transformer", func() { It("marks header as checked even if no logs were returned", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) fetcher.SetFetchedLogs([]types.Log{}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Converter: &converter, Fetcher: &fetcher, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -163,12 +163,12 @@ var _ = Describe("Vat move transformer", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) repository.SetCheckedHeaderError(fakes.FakeError) fetcher.SetFetchedLogs([]types.Log{}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Converter: &converter, Fetcher: &fetcher, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -179,30 +179,30 @@ var _ = Describe("Vat move transformer", func() { It("persists vat move model", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) - Expect(repository.PassedModels).To(Equal([]vat_move.VatMoveModel{test_data.VatMoveModel})) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.VatMoveModel})) }) It("returns error if repository returns error for create", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := vat_move.VatMoveTransformer{ + transformer := factories.Transformer{ Config: config, Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) err := transformer.Execute() From 46d8ea8d97024154ce39ea1746fca26a2d0ff01e Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 9 Oct 2018 16:37:02 +0200 Subject: [PATCH 05/36] Snag transformer topics from config in factory --- pkg/transformers/factories/transformer.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index a1d30b11..3eecced0 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -57,13 +57,19 @@ func (transformer Transformer) Execute() error { log.Printf("Fetching vat move event logs for %d headers \n", len(missingHeaders)) for _, header := range missingHeaders { - // TODO Needs signature in config - topics := [][]common.Hash{{common.HexToHash(shared.VatMoveSignature)}} + // Grab topics from config + var topics [][]common.Hash + for _, topic := range transformer.Config.Topics { + topics = append(topics, []common.Hash{common.HexToHash(topic)}) + } + + // Fetch the missing logs for a given header matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topics, header.BlockNumber) if err != nil { return err } + // No matching logs, mark the header as checked for this type of logs if len(matchingLogs) < 1 { err := transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { @@ -76,7 +82,8 @@ func (transformer Transformer) Execute() error { return err } - // Can't assert the whole collection, need to wash types individually + // Can't assert a whole collection, wash types individually for Create, + // which needs interface{} var typelessModels []interface{} for _, m := range models { typelessModels = append(typelessModels, m.(interface{})) From 32654ffabcb589a63a1b06b5451e534df63559b6 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 11 Oct 2018 15:13:22 +0200 Subject: [PATCH 06/36] Add transformer name to config for logging in factory transformer --- pkg/transformers/factories/transformer.go | 2 +- pkg/transformers/shared/transformer.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 3eecced0..f4a2420c 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -55,7 +55,7 @@ func (transformer Transformer) Execute() error { return err } - log.Printf("Fetching vat move event logs for %d headers \n", len(missingHeaders)) + log.Printf("Fetching %v event logs for %d headers \n", transformer.Config.TransformerName, len(missingHeaders)) for _, header := range missingHeaders { // Grab topics from config var topics [][]common.Hash diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 1b07026e..acede32c 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -28,6 +28,7 @@ type Transformer interface { type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer type TransformerConfig struct { + TransformerName string ContractAddresses []string ContractAbi string Topics []string From 177d212cc469fbd2d3df76190dd5087ace6f40c8 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 11 Oct 2018 15:55:39 +0200 Subject: [PATCH 07/36] Add SetDB test for vat move --- pkg/transformers/test_data/mocks/vat_move/repository.go | 4 +--- pkg/transformers/vat_move/config.go | 1 + pkg/transformers/vat_move/repository_test.go | 9 +++++++++ pkg/transformers/vat_move/transformer_test.go | 4 ++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/transformers/test_data/mocks/vat_move/repository.go b/pkg/transformers/test_data/mocks/vat_move/repository.go index 3b40753f..46c3eb02 100644 --- a/pkg/transformers/test_data/mocks/vat_move/repository.go +++ b/pkg/transformers/test_data/mocks/vat_move/repository.go @@ -64,6 +64,4 @@ func (repository *MockVatMoveRepository) SetCheckedHeaderError(e error) { repository.CheckedHeaderError = e } -func (repository *MockVatMoveRepository) SetDB(db *postgres.DB) { - // TODO Implement passed DB -} \ No newline at end of file +func (repository *MockVatMoveRepository) SetDB(db *postgres.DB) {} diff --git a/pkg/transformers/vat_move/config.go b/pkg/transformers/vat_move/config.go index 32077783..a98bc52a 100644 --- a/pkg/transformers/vat_move/config.go +++ b/pkg/transformers/vat_move/config.go @@ -19,6 +19,7 @@ import ( ) var VatMoveConfig = shared.TransformerConfig{ + TransformerName: "vat move", ContractAddresses: []string{shared.VatContractAddress}, ContractAbi: shared.VatABI, Topics: []string{shared.VatMoveSignature}, diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index 978de862..986cc4d1 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -190,4 +190,13 @@ var _ = Describe("Vat Move", func() { Expect(headerChecked).To(BeTrue()) }) }) + + Describe("SetDB", func() { + It("sets the repository db", func() { + vatMoveRepository.DB = nil + Expect(vatMoveRepository.DB).To(BeNil()) + vatMoveRepository.SetDB(db) + Expect(vatMoveRepository.DB).To(Equal(db)) + }) + }) }) diff --git a/pkg/transformers/vat_move/transformer_test.go b/pkg/transformers/vat_move/transformer_test.go index 9b63cc06..e49dacc0 100644 --- a/pkg/transformers/vat_move/transformer_test.go +++ b/pkg/transformers/vat_move/transformer_test.go @@ -47,10 +47,10 @@ var _ = Describe("Vat move transformer", func() { It("gets missing headers for block numbers specified in config", func() { transformer := factories.Transformer{ - Config: vat_move.VatMoveConfig, + Config: config, Converter: &converter, Repository: &repository, - Fetcher: &fetcher, + Fetcher: &fetcher, }.NewTransformer(nil, nil) err := transformer.Execute() From fedbce80f6a59b25e176e6aaf4a28d0df82ff509 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 11 Oct 2018 16:06:37 +0200 Subject: [PATCH 08/36] Refactor vat init transformer to use factory --- pkg/transformers/transformers.go | 18 ++++-- pkg/transformers/vat_init/config.go | 1 + pkg/transformers/vat_init/converter.go | 9 +-- pkg/transformers/vat_init/repository.go | 40 ++++++------ pkg/transformers/vat_init/transformer.go | 78 ------------------------ 5 files changed, 34 insertions(+), 112 deletions(-) delete mode 100644 pkg/transformers/vat_init/transformer.go diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 313b7bdf..6e4bfc0f 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -72,12 +72,18 @@ var ( PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer - VatInitTransformerInitializer = vat_init.VatInitTransformerInitializer{Config: vat_init.VatInitConfig}.NewVatInitTransformer - VatMoveTransformerInitializer = factories.Transformer{ - Config: vat_move.VatMoveConfig, - Converter: &vat_move.VatMoveConverter{}, - Repository: &vat_move.VatMoveRepository{}, - Fetcher: &shared.Fetcher{}, + VatInitTransformerInitializer = factories.Transformer{ + Config: vat_init.VatInitConfig, + Converter: &vat_init.VatInitConverter{}, + Repository: &vat_init.VatInitRepository{DB: nil}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + + VatMoveTransformerInitializer = factories.Transformer{ + Config: vat_move.VatMoveConfig, + Converter: &vat_move.VatMoveConverter{}, + Repository: &vat_move.VatMoveRepository{DB: nil}, + Fetcher: &shared.Fetcher{}, }.NewTransformer VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer diff --git a/pkg/transformers/vat_init/config.go b/pkg/transformers/vat_init/config.go index dcc9a03e..2f9abf25 100644 --- a/pkg/transformers/vat_init/config.go +++ b/pkg/transformers/vat_init/config.go @@ -19,6 +19,7 @@ import ( ) var VatInitConfig = shared.TransformerConfig{ + TransformerName: "vat init", ContractAddresses: []string{shared.VatContractAddress}, ContractAbi: shared.VatABI, Topics: []string{shared.VatInitSignature}, diff --git a/pkg/transformers/vat_init/converter.go b/pkg/transformers/vat_init/converter.go index 7fc510b5..88f40fdf 100644 --- a/pkg/transformers/vat_init/converter.go +++ b/pkg/transformers/vat_init/converter.go @@ -18,18 +18,13 @@ import ( "bytes" "encoding/json" "errors" - "github.com/ethereum/go-ethereum/core/types" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]VatInitModel, error) -} - type VatInitConverter struct{} -func (VatInitConverter) ToModels(ethLogs []types.Log) ([]VatInitModel, error) { - var models []VatInitModel +func (VatInitConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index f17d91c5..815593a7 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -17,40 +17,34 @@ package vat_init import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "log" ) -type Repository interface { - Create(headerID int64, models []VatInitModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type VatInitRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewVatInitRepository(db *postgres.DB) VatInitRepository { - return VatInitRepository{ - db: db, - } -} - -func (repository VatInitRepository) Create(headerID int64, models []VatInitModel) error { - tx, err := repository.db.Begin() +func (repository VatInitRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + var vatInit VatInitModel for _, model := range models { + vatInit = model.(VatInitModel) + log.Printf("VatInit model: %v", vatInit) _, err = tx.Exec( - `INSERT into maker.vat_init (header_id, ilk, tx_idx, raw_log) - VALUES($1, $2, $3, $4)`, - headerID, model.Ilk, model.TransactionIndex, model.Raw, + `INSERT INTO maker.vat_init (header_id, ilk, tx_idx, raw_log) + VALUES($1, $2, $3, $4)`, + headerID, vatInit.Ilk, vatInit.TransactionIndex, vatInit.Raw, ) if err != nil { tx.Rollback() + log.Printf("Error: %v \n", err) return err } } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked) VALUES($1, $2) ON CONFLICT (header_id) DO @@ -64,7 +58,7 @@ func (repository VatInitRepository) Create(headerID int64, models []VatInitModel } func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET vat_init_checked = $2`, headerID, true) @@ -73,7 +67,7 @@ func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -83,7 +77,11 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *VatInitRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/vat_init/transformer.go b/pkg/transformers/vat_init/transformer.go deleted file mode 100644 index 2404f67b..00000000 --- a/pkg/transformers/vat_init/transformer.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package vat_init - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type VatInitTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer VatInitTransformerInitializer) NewVatInitTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := VatInitConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewVatInitRepository(db) - return VatInitTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -type VatInitTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -func (transformer VatInitTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching vat init event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.VatInitSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(VatInitConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - models, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, models) - if err != nil { - return err - } - } - return nil -} From 719ffd581975485a1d33f47e42934b8b4d66cc3f Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 11 Oct 2018 16:07:00 +0200 Subject: [PATCH 09/36] Update vat init tests for factory --- .../test_data/mocks/vat_init/converter.go | 11 +- .../test_data/mocks/vat_init/repository.go | 30 ++-- pkg/transformers/vat_init/repository_test.go | 51 +++++- pkg/transformers/vat_init/transformer_test.go | 160 +++++++++--------- 4 files changed, 144 insertions(+), 108 deletions(-) diff --git a/pkg/transformers/test_data/mocks/vat_init/converter.go b/pkg/transformers/test_data/mocks/vat_init/converter.go index 3b8b9ccd..dcc0b3a1 100644 --- a/pkg/transformers/test_data/mocks/vat_init/converter.go +++ b/pkg/transformers/test_data/mocks/vat_init/converter.go @@ -18,19 +18,18 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" - "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" ) type MockVatInitConverter struct { - converterErr error - PassedLogs []types.Log + converterError error + PassedLogs []types.Log } -func (converter *MockVatInitConverter) ToModels(ethLogs []types.Log) ([]vat_init.VatInitModel, error) { +func (converter *MockVatInitConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []vat_init.VatInitModel{test_data.VatInitModel}, converter.converterErr + return []interface{}{test_data.VatInitModel}, converter.converterError } func (converter *MockVatInitConverter) SetConverterError(e error) { - converter.converterErr = e + converter.converterError = e } diff --git a/pkg/transformers/test_data/mocks/vat_init/repository.go b/pkg/transformers/test_data/mocks/vat_init/repository.go index 7a9978b6..298c941d 100644 --- a/pkg/transformers/test_data/mocks/vat_init/repository.go +++ b/pkg/transformers/test_data/mocks/vat_init/repository.go @@ -16,46 +16,46 @@ package vat_init import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" ) type MockVatInitRepository struct { - createErr error - markHeaderCheckedErr error + createError error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []vat_init.VatInitModel + PassedModels []interface{} } -func (repository *MockVatInitRepository) Create(headerID int64, models []vat_init.VatInitModel) error { +func (repository *MockVatInitRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models - return repository.createErr + return repository.createError } func (repository *MockVatInitRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockVatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } -func (repository *MockVatInitRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e +func (repository *MockVatInitRepository) SetMarkHeaderCheckedError(e error) { + repository.markHeaderCheckedError = e } -func (repository *MockVatInitRepository) SetMissingHeadersErr(e error) { - repository.missingHeadersErr = e +func (repository *MockVatInitRepository) SetMissingHeadersError(e error) { + repository.missingHeadersError = e } func (repository *MockVatInitRepository) SetMissingHeaders(headers []core.Header) { @@ -63,9 +63,11 @@ func (repository *MockVatInitRepository) SetMissingHeaders(headers []core.Header } func (repository *MockVatInitRepository) SetCreateError(e error) { - repository.createErr = e + repository.createError = e } func (repository *MockVatInitRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockVatInitRepository) SetDB(db *postgres.DB) {} diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index ec84008d..7ce6a647 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -32,7 +32,7 @@ import ( var _ = Describe("Vat init repository", func() { var ( db *postgres.DB - vatInitRepository vat_init.Repository + vatInitRepository vat_init.VatInitRepository headerRepository repositories.HeaderRepository err error ) @@ -40,7 +40,7 @@ var _ = Describe("Vat init repository", func() { BeforeEach(func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) - vatInitRepository = vat_init.NewVatInitRepository(db) + vatInitRepository = vat_init.VatInitRepository{DB: db} headerRepository = repositories.NewHeaderRepository(db) }) @@ -51,13 +51,13 @@ var _ = Describe("Vat init repository", func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel}) + err = vatInitRepository.Create(headerID, []interface{}{test_data.VatInitModel}) Expect(err).NotTo(HaveOccurred()) }) It("adds a vat event", func() { var dbVatInit vat_init.VatInitModel - err = db.Get(&dbVatInit, `SELECT ilk,tx_idx, raw_log FROM maker.vat_init WHERE header_id = $1`, headerID) + err = db.Get(&dbVatInit, `SELECT ilk, tx_idx, raw_log FROM maker.vat_init WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) Expect(dbVatInit.Ilk).To(Equal(test_data.VatInitModel.Ilk)) Expect(dbVatInit.TransactionIndex).To(Equal(test_data.VatInitModel.TransactionIndex)) @@ -65,7 +65,10 @@ var _ = Describe("Vat init repository", func() { }) It("does not duplicate vat events", func() { - err = vatInitRepository.Create(headerID, []vat_init.VatInitModel{test_data.VatInitModel}) + err = vatInitRepository.Create(headerID, []interface{}{test_data.VatInitModel}) + Expect(err).NotTo(HaveOccurred()) + + err = vatInitRepository.Create(headerID, []interface{}{test_data.VatInitModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -90,12 +93,20 @@ var _ = Describe("Vat init repository", func() { }) Describe("MarkHeaderChecked", func() { - var headerID int64 + var ( + db *postgres.DB + vatInitRepository vat_init.VatInitRepository + err error + headerID int64 + ) BeforeEach(func() { - headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + db = test_config.NewTestDB(core.Node{}) + test_config.CleanTestDB(db) + headerRepository := repositories.NewHeaderRepository(db) + headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) Expect(err).NotTo(HaveOccurred()) - + vatInitRepository = vat_init.VatInitRepository{DB: db} }) It("creates a row for a new headerID", func() { @@ -155,6 +166,16 @@ var _ = Describe("Vat init repository", func() { }) It("only treats headers as checked if drip drip logs have been checked", func() { + startingBlockNumber := int64(1) + vatInitBlockNumber := int64(2) + endingBlockNumber := int64(3) + blockNumbers = []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1} + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) Expect(err).NotTo(HaveOccurred()) @@ -174,7 +195,9 @@ var _ = Describe("Vat init repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - vatInitRepositoryTwo := vat_init.NewVatInitRepository(dbTwo) + + vatInitRepository := vat_init.VatInitRepository{DB: db} + vatInitRepositoryTwo := vat_init.VatInitRepository{DB: dbTwo} err := vatInitRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -187,4 +210,14 @@ var _ = Describe("Vat init repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) + + Describe("SetDB", func() { + It("sets the repository db", func() { + db := test_config.NewTestDB(core.Node{}) + vatInitRepository := vat_init.VatInitRepository{DB: nil} + Expect(vatInitRepository.DB).To(BeNil()) + vatInitRepository.SetDB(db) + Expect(vatInitRepository.DB).To(Equal(db)) + }) + }) }) diff --git a/pkg/transformers/vat_init/transformer_test.go b/pkg/transformers/vat_init/transformer_test.go index 95d2acab..44b0cd82 100644 --- a/pkg/transformers/vat_init/transformer_test.go +++ b/pkg/transformers/vat_init/transformer_test.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -30,14 +31,30 @@ import ( ) var _ = Describe("Vat init transformer", func() { + var ( + config = vat_init.VatInitConfig + converter vat_init_mocks.MockVatInitConverter + repository vat_init_mocks.MockVatInitRepository + fetcher mocks.MockLogFetcher + headerOne core.Header + headerTwo core.Header + ) + + BeforeEach(func() { + converter = vat_init_mocks.MockVatInitConverter{} + repository = vat_init_mocks.MockVatInitRepository{} + fetcher = mocks.MockLogFetcher{} + headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + }) + It("gets missing headers for block numbers specified in config", func() { - repository := &vat_init_mocks.MockVatInitRepository{} - transformer := vat_init.VatInitTransformer{ - Config: vat_init.VatInitConfig, + transformer := factories.Transformer{ + Config: config, Fetcher: &mocks.MockLogFetcher{}, Converter: &vat_init_mocks.MockVatInitConverter{}, - Repository: repository, - } + Repository: &repository, + }.NewTransformer(nil, nil) err := transformer.Execute() @@ -47,12 +64,12 @@ var _ = Describe("Vat init transformer", func() { }) It("returns error if repository returns error for missing headers", func() { - repository := &vat_init_mocks.MockVatInitRepository{} - repository.SetMissingHeadersErr(fakes.FakeError) - transformer := vat_init.VatInitTransformer{ + repository.SetMissingHeadersError(fakes.FakeError) + transformer := factories.Transformer{ + Config: config, Fetcher: &mocks.MockLogFetcher{}, Converter: &vat_init_mocks.MockVatInitConverter{}, - Repository: repository, + Repository: &repository, } err := transformer.Execute() @@ -62,32 +79,31 @@ var _ = Describe("Vat init transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &vat_init_mocks.MockVatInitRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := vat_init.VatInitTransformer{ - Fetcher: fetcher, - Converter: &vat_init_mocks.MockVatInitConverter{}, - Repository: repository, + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{vat_init.VatInitConfig.ContractAddresses, vat_init.VatInitConfig.ContractAddresses})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) + Expect(fetcher.FetchedContractAddresses).To(Equal( + [][]string{transformer.Config.ContractAddresses, transformer.Config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatInitSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &vat_init_mocks.MockVatInitRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vat_init.VatInitTransformer{ - Fetcher: fetcher, - Converter: &vat_init_mocks.MockVatInitConverter{}, - Repository: repository, + repository.SetMissingHeaders([]core.Header{headerOne}) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() @@ -97,33 +113,28 @@ var _ = Describe("Vat init transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &vat_init_mocks.MockVatInitConverter{} - mockRepository := &vat_init_mocks.MockVatInitRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := vat_init.VatInitTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, + repository.SetMissingHeaders([]core.Header{headerOne}) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &vat_init_mocks.MockVatInitConverter{} - mockRepository := &vat_init_mocks.MockVatInitRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := vat_init.VatInitTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedError(fakes.FakeError) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() @@ -133,15 +144,13 @@ var _ = Describe("Vat init transformer", func() { }) It("converts matching logs", func() { - converter := &vat_init_mocks.MockVatInitConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) - repository := &vat_init_mocks.MockVatInitRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vat_init.VatInitTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, + repository.SetMissingHeaders([]core.Header{headerOne}) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() @@ -151,16 +160,14 @@ var _ = Describe("Vat init transformer", func() { }) It("returns error if converter returns error", func() { - converter := &vat_init_mocks.MockVatInitConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) - repository := &vat_init_mocks.MockVatInitRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vat_init.VatInitTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, + repository.SetMissingHeaders([]core.Header{headerOne}) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() @@ -170,36 +177,31 @@ var _ = Describe("Vat init transformer", func() { }) It("persists vat init model", func() { - converter := &vat_init_mocks.MockVatInitConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) - repository := &vat_init_mocks.MockVatInitRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := vat_init.VatInitTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, + repository.SetMissingHeaders([]core.Header{headerOne}) + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]vat_init.VatInitModel{test_data.VatInitModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.VatInitModel})) }) It("returns error if repository returns error for create", func() { - converter := &vat_init_mocks.MockVatInitConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) - repository := &vat_init_mocks.MockVatInitRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := vat_init.VatInitTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, + transformer := factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, } err := transformer.Execute() From c4d3783f8b91a2426a709b992c8caf0155e37b56 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 12 Oct 2018 14:38:45 +0200 Subject: [PATCH 10/36] Expand error logging in transformer --- pkg/transformers/factories/transformer.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index f4a2420c..3be68c56 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -50,12 +50,14 @@ func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChai } func (transformer Transformer) Execute() error { + transformerName := transformer.Config.TransformerName missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) if err != nil { + log.Printf("Error fetching mising headers in %v transformer: %v \n", transformerName, err) return err } - log.Printf("Fetching %v event logs for %d headers \n", transformer.Config.TransformerName, len(missingHeaders)) + log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders)) for _, header := range missingHeaders { // Grab topics from config var topics [][]common.Hash @@ -66,6 +68,7 @@ func (transformer Transformer) Execute() error { // Fetch the missing logs for a given header matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topics, header.BlockNumber) if err != nil { + log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) return err } @@ -73,12 +76,14 @@ func (transformer Transformer) Execute() error { if len(matchingLogs) < 1 { err := transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { + log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err } } models, err := transformer.Converter.ToModels(matchingLogs) if err != nil { + log.Printf("Error converting logs in %v: %v", transformerName, err) return err } @@ -91,6 +96,7 @@ func (transformer Transformer) Execute() error { err = transformer.Repository.Create(header.Id, typelessModels) if err != nil { + log.Printf("Error persisting %v record: %v", transformerName, err) return err } } From 4d36f6200d396c204e8b1fb1499d78f4b6c9b14a Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 12 Oct 2018 15:05:43 +0200 Subject: [PATCH 11/36] Further refactor move and init tests --- pkg/transformers/transformers.go | 4 +- pkg/transformers/vat_init/converter_test.go | 12 +-- pkg/transformers/vat_init/transformer_test.go | 82 ++++--------------- pkg/transformers/vat_move/converter_test.go | 12 +-- pkg/transformers/vat_move/transformer_test.go | 71 ++-------------- 5 files changed, 39 insertions(+), 142 deletions(-) diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 6e4bfc0f..6cd37253 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -75,14 +75,14 @@ var ( VatInitTransformerInitializer = factories.Transformer{ Config: vat_init.VatInitConfig, Converter: &vat_init.VatInitConverter{}, - Repository: &vat_init.VatInitRepository{DB: nil}, + Repository: &vat_init.VatInitRepository{}, Fetcher: &shared.Fetcher{}, }.NewTransformer VatMoveTransformerInitializer = factories.Transformer{ Config: vat_move.VatMoveConfig, Converter: &vat_move.VatMoveConverter{}, - Repository: &vat_move.VatMoveRepository{DB: nil}, + Repository: &vat_move.VatMoveRepository{}, Fetcher: &shared.Fetcher{}, }.NewTransformer VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer diff --git a/pkg/transformers/vat_init/converter_test.go b/pkg/transformers/vat_init/converter_test.go index f07625ba..f7e158bc 100644 --- a/pkg/transformers/vat_init/converter_test.go +++ b/pkg/transformers/vat_init/converter_test.go @@ -24,18 +24,20 @@ import ( ) var _ = Describe("Vat init converter", func() { - It("returns err if log missing topics", func() { - converter := vat_init.VatInitConverter{} - badLog := types.Log{} + var converter vat_init.VatInitConverter + BeforeEach(func() { + converter = vat_init.VatInitConverter{} + }) + + It("returns err if log missing topics", func() { + badLog := types.Log{} _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) It("converts a log to an model", func() { - converter := vat_init.VatInitConverter{} - models, err := converter.ToModels([]types.Log{test_data.EthVatInitLog}) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/vat_init/transformer_test.go b/pkg/transformers/vat_init/transformer_test.go index 44b0cd82..d80a1334 100644 --- a/pkg/transformers/vat_init/transformer_test.go +++ b/pkg/transformers/vat_init/transformer_test.go @@ -32,12 +32,13 @@ import ( var _ = Describe("Vat init transformer", func() { var ( - config = vat_init.VatInitConfig - converter vat_init_mocks.MockVatInitConverter - repository vat_init_mocks.MockVatInitRepository - fetcher mocks.MockLogFetcher - headerOne core.Header - headerTwo core.Header + config = vat_init.VatInitConfig + converter vat_init_mocks.MockVatInitConverter + repository vat_init_mocks.MockVatInitRepository + fetcher mocks.MockLogFetcher + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header ) BeforeEach(func() { @@ -46,16 +47,15 @@ var _ = Describe("Vat init transformer", func() { fetcher = mocks.MockLogFetcher{} headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + transformer = factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, + }.NewTransformer(nil, nil) }) It("gets missing headers for block numbers specified in config", func() { - transformer := factories.Transformer{ - Config: config, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &vat_init_mocks.MockVatInitConverter{}, - Repository: &repository, - }.NewTransformer(nil, nil) - err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -65,12 +65,6 @@ var _ = Describe("Vat init transformer", func() { It("returns error if repository returns error for missing headers", func() { repository.SetMissingHeadersError(fakes.FakeError) - transformer := factories.Transformer{ - Config: config, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &vat_init_mocks.MockVatInitConverter{}, - Repository: &repository, - } err := transformer.Execute() @@ -80,31 +74,19 @@ var _ = Describe("Vat init transformer", func() { It("fetches logs for missing headers", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) Expect(fetcher.FetchedContractAddresses).To(Equal( - [][]string{transformer.Config.ContractAddresses, transformer.Config.ContractAddresses})) + [][]string{config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.VatInitSignature)}})) }) It("returns error if fetcher returns error", func() { fetcher.SetFetcherError(fakes.FakeError) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() @@ -114,12 +96,6 @@ var _ = Describe("Vat init transformer", func() { It("marks header checked if no logs returned", func() { repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() @@ -130,12 +106,6 @@ var _ = Describe("Vat init transformer", func() { It("returns error if marking header checked returns err", func() { repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetMarkHeaderCheckedError(fakes.FakeError) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() @@ -146,12 +116,6 @@ var _ = Describe("Vat init transformer", func() { It("converts matching logs", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() @@ -163,12 +127,6 @@ var _ = Describe("Vat init transformer", func() { converter.SetConverterError(fakes.FakeError) fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() @@ -179,12 +137,6 @@ var _ = Describe("Vat init transformer", func() { It("persists vat init model", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() @@ -197,12 +149,6 @@ var _ = Describe("Vat init transformer", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatInitLog}) repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - } err := transformer.Execute() diff --git a/pkg/transformers/vat_move/converter_test.go b/pkg/transformers/vat_move/converter_test.go index b4246865..c712cbd9 100644 --- a/pkg/transformers/vat_move/converter_test.go +++ b/pkg/transformers/vat_move/converter_test.go @@ -24,18 +24,20 @@ import ( ) var _ = Describe("Vat move converter", func() { - It("returns err if logs are missing topics", func() { - converter := vat_move.VatMoveConverter{} - badLog := types.Log{} + var converter vat_move.VatMoveConverter + BeforeEach(func() { + converter = vat_move.VatMoveConverter{} + }) + + It("returns err if logs are missing topics", func() { + badLog := types.Log{} _, err := converter.ToModels([]types.Log{badLog}) Expect(err).To(HaveOccurred()) }) It("converts a log to a model", func() { - converter := vat_move.VatMoveConverter{} - models, err := converter.ToModels([]types.Log{test_data.EthVatMoveLog}) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/vat_move/transformer_test.go b/pkg/transformers/vat_move/transformer_test.go index e49dacc0..0e35f6d3 100644 --- a/pkg/transformers/vat_move/transformer_test.go +++ b/pkg/transformers/vat_move/transformer_test.go @@ -30,10 +30,11 @@ import ( ) var _ = Describe("Vat move transformer", func() { + var config = vat_move.VatMoveConfig var fetcher mocks.MockLogFetcher var converter vat_move_mocks.MockVatMoveConverter var repository vat_move_mocks.MockVatMoveRepository - var config = vat_move.VatMoveConfig + var transformer shared.Transformer var headerOne core.Header var headerTwo core.Header @@ -43,16 +44,15 @@ var _ = Describe("Vat move transformer", func() { repository = vat_move_mocks.MockVatMoveRepository{} headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + transformer = factories.Transformer{ + Config: config, + Converter: &converter, + Fetcher: &fetcher, + Repository: &repository, + }.NewTransformer(nil, nil) }) It("gets missing headers for block numbers specified in config", func() { - transformer := factories.Transformer{ - Config: config, - Converter: &converter, - Repository: &repository, - Fetcher: &fetcher, - }.NewTransformer(nil, nil) - err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -62,12 +62,7 @@ var _ = Describe("Vat move transformer", func() { It("returns error if repository returns error for missing headers", func() { repository.SetMissingHeadersError(fakes.FakeError) - transformer := factories.Transformer{ - Config: config, - Converter: &converter, - Fetcher: &fetcher, - Repository: &repository, - }.NewTransformer(nil, nil) + err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -76,12 +71,6 @@ var _ = Describe("Vat move transformer", func() { It("fetches logs for missing headers", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() @@ -97,12 +86,6 @@ var _ = Describe("Vat move transformer", func() { It("returns error if fetcher returns error", func() { fetcher.SetFetcherError(fakes.FakeError) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() @@ -113,12 +96,6 @@ var _ = Describe("Vat move transformer", func() { It("converts matching logs", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() @@ -130,12 +107,6 @@ var _ = Describe("Vat move transformer", func() { converter.SetConverterError(fakes.FakeError) fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() @@ -146,12 +117,6 @@ var _ = Describe("Vat move transformer", func() { It("marks header as checked even if no logs were returned", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) fetcher.SetFetchedLogs([]types.Log{}) - transformer := factories.Transformer{ - Config: config, - Converter: &converter, - Fetcher: &fetcher, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -163,12 +128,6 @@ var _ = Describe("Vat move transformer", func() { repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) repository.SetCheckedHeaderError(fakes.FakeError) fetcher.SetFetchedLogs([]types.Log{}) - transformer := factories.Transformer{ - Config: config, - Converter: &converter, - Fetcher: &fetcher, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() @@ -179,12 +138,6 @@ var _ = Describe("Vat move transformer", func() { It("persists vat move model", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() @@ -197,12 +150,6 @@ var _ = Describe("Vat move transformer", func() { fetcher.SetFetchedLogs([]types.Log{test_data.EthVatMoveLog}) repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := factories.Transformer{ - Config: config, - Fetcher: &fetcher, - Converter: &converter, - Repository: &repository, - }.NewTransformer(nil, nil) err := transformer.Execute() From a4df8f348d34e37c5c76a648e3226bc365c852cd Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 12 Oct 2018 16:17:49 +0200 Subject: [PATCH 12/36] Refactor Tend transformer using factory --- pkg/transformers/tend/config.go | 1 + pkg/transformers/tend/converter.go | 10 +-- pkg/transformers/tend/converter_test.go | 2 +- pkg/transformers/tend/repository.go | 44 ++++++------ pkg/transformers/tend/repository_test.go | 24 ++++--- pkg/transformers/tend/transformer.go | 88 ----------------------- pkg/transformers/tend/transformer_test.go | 73 ++++++++----------- pkg/transformers/transformers.go | 10 ++- 8 files changed, 76 insertions(+), 176 deletions(-) delete mode 100644 pkg/transformers/tend/transformer.go diff --git a/pkg/transformers/tend/config.go b/pkg/transformers/tend/config.go index 7ec52f9b..03a15e70 100644 --- a/pkg/transformers/tend/config.go +++ b/pkg/transformers/tend/config.go @@ -17,6 +17,7 @@ package tend import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var TendConfig = shared.TransformerConfig{ + TransformerName: "tend", ContractAddresses: []string{shared.FlipperContractAddress}, ContractAbi: shared.FlipperABI, Topics: []string{shared.TendFunctionSignature}, diff --git a/pkg/transformers/tend/converter.go b/pkg/transformers/tend/converter.go index dd989d08..52ace3c1 100644 --- a/pkg/transformers/tend/converter.go +++ b/pkg/transformers/tend/converter.go @@ -23,17 +23,9 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]TendModel, error) -} - type TendConverter struct{} -func NewTendConverter() TendConverter { - return TendConverter{} -} - -func (c TendConverter) ToModels(ethLogs []types.Log) (results []TendModel, err error) { +func (TendConverter) ToModels(ethLogs []types.Log) (results []interface{}, err error) { for _, ethLog := range ethLogs { err := validateLog(ethLog) if err != nil { diff --git a/pkg/transformers/tend/converter_test.go b/pkg/transformers/tend/converter_test.go index db53220d..ec07fec3 100644 --- a/pkg/transformers/tend/converter_test.go +++ b/pkg/transformers/tend/converter_test.go @@ -28,7 +28,7 @@ var _ = Describe("Tend TendConverter", func() { var converter tend.TendConverter BeforeEach(func() { - converter = tend.NewTendConverter() + converter = tend.TendConverter{} }) Describe("ToModels", func() { diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 84cf2d22..19166a84 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -19,31 +19,24 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerId int64, tends []TendModel) error - MarkHeaderChecked(headerId int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type TendRepository struct { DB *postgres.DB } -func NewTendRepository(db *postgres.DB) TendRepository { - return TendRepository{DB: db} -} - -func (r TendRepository) Create(headerId int64, tends []TendModel) error { - tx, err := r.DB.Begin() +func (repository TendRepository) Create(headerId int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } - for _, tend := range tends { + var tend TendModel + for _, model := range models { + tend = model.(TendModel) _, err = tx.Exec( `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6, $7, $8)`, + VALUES($1, $2, $3, $4, $5, $6, $7, $8)`, headerId, tend.BidId, tend.Lot, tend.Bid, tend.Guy, tend.Tic, tend.TransactionIndex, tend.Raw, ) + if err != nil { tx.Rollback() return err @@ -53,6 +46,7 @@ func (r TendRepository) Create(headerId int64, tends []TendModel) error { VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET tend_checked = $2`, headerId, true) + if err != nil { tx.Rollback() return err @@ -60,28 +54,32 @@ func (r TendRepository) Create(headerId int64, tends []TendModel) error { return tx.Commit() } -func (r TendRepository) MarkHeaderChecked(headerId int64) error { - _, err := r.DB.Exec(`INSERT INTO public.checked_headers (header_id, tend_checked) +func (repository TendRepository) MarkHeaderChecked(headerId int64) error { + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, tend_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET tend_checked = $2`, headerId, true) return err } -func (r TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { +func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := r.DB.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id + LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR tend_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - r.DB.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *TendRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index 943c96cc..a6d43f58 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -32,6 +32,7 @@ var _ = Describe("TendRepository", func() { db *postgres.DB tendRepository tend.TendRepository headerRepository repositories.HeaderRepository + headerId int64 err error ) @@ -40,17 +41,15 @@ var _ = Describe("TendRepository", func() { db = test_config.NewTestDB(node) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - tendRepository = tend.NewTendRepository(db) + headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + Expect(err).NotTo(HaveOccurred()) + + tendRepository = tend.TendRepository{DB: db} }) Describe("Create", func() { - var headerId int64 - - BeforeEach(func() { - headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) - Expect(err).NotTo(HaveOccurred()) - - err := tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel}) + It("persists a tend record", func() { + err := tendRepository.Create(headerId, []interface{}{test_data.TendModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -81,13 +80,18 @@ var _ = Describe("TendRepository", func() { }) It("returns an error if inserting a tend record fails", func() { - err = tendRepository.Create(headerId, []tend.TendModel{test_data.TendModel}) + err := tendRepository.Create(headerId, []interface{}{test_data.TendModel}) + Expect(err).NotTo(HaveOccurred()) + err = tendRepository.Create(headerId, []interface{}{test_data.TendModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("deletes the tend record if its corresponding header record is deleted", func() { + err := tendRepository.Create(headerId, []interface{}{test_data.TendModel}) + Expect(err).NotTo(HaveOccurred()) + var count int err = db.QueryRow(`SELECT count(*) from maker.tend`).Scan(&count) Expect(err).NotTo(HaveOccurred()) @@ -182,7 +186,7 @@ var _ = Describe("TendRepository", func() { node2 := core.Node{} db2 := test_config.NewTestDB(node2) headerRepository2 := repositories.NewHeaderRepository(db2) - tendRepository2 := tend.NewTendRepository(db2) + tendRepository2 := tend.TendRepository{DB: db2} for _, number := range []int64{startingBlock, tendBlock, endingBlock} { headerRepository2.CreateOrUpdateHeader(fakes.GetFakeHeader(number)) diff --git a/pkg/transformers/tend/transformer.go b/pkg/transformers/tend/transformer.go deleted file mode 100644 index ea8ce4b8..00000000 --- a/pkg/transformers/tend/transformer.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tend - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type TendTransformer struct { - Repository Repository - Fetcher shared.LogFetcher - Converter Converter - Config shared.TransformerConfig -} - -type TendTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (i TendTransformerInitializer) NewTendTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := NewTendConverter() - fetcher := shared.NewFetcher(blockChain) - repository := NewTendRepository(db) - return TendTransformer{ - Fetcher: fetcher, - Repository: repository, - Converter: converter, - Config: i.Config, - } -} - -func (t TendTransformer) Execute() error { - config := t.Config - topics := [][]common.Hash{{common.HexToHash(shared.TendFunctionSignature)}} - - missingHeaders, err := t.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber) - if err != nil { - log.Println("Error fetching missing headers:", err) - return err - } - - log.Printf("Fetching tend event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - ethLogs, err := t.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber) - if err != nil { - log.Println("Error fetching matching logs:", err) - return err - } - if len(ethLogs) < 1 { - err := t.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - - models, err := t.Converter.ToModels(ethLogs) - if err != nil { - log.Println("Error converting logs:", err) - return err - } - - err = t.Repository.Create(header.Id, models) - if err != nil { - log.Println("Error persisting tend record:", err) - return err - } - } - - return nil -} diff --git a/pkg/transformers/tend/transformer_test.go b/pkg/transformers/tend/transformer_test.go index 57181cb3..271f040b 100644 --- a/pkg/transformers/tend/transformer_test.go +++ b/pkg/transformers/tend/transformer_test.go @@ -15,7 +15,7 @@ package tend_test import ( - "math/rand" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" @@ -32,24 +32,28 @@ import ( ) var _ = Describe("Tend Transformer", func() { - var repository tend_mocks.MockTendRepository - var fetcher mocks.MockLogFetcher - var converter tend_mocks.MockTendConverter - var transformer tend.TendTransformer - var blockNumber1 = rand.Int63() - var blockNumber2 = rand.Int63() + var ( + config = tend.TendConfig + converter tend_mocks.MockTendConverter + repository tend_mocks.MockTendRepository + fetcher mocks.MockLogFetcher + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) BeforeEach(func() { + converter = tend_mocks.MockTendConverter{} repository = tend_mocks.MockTendRepository{} fetcher = mocks.MockLogFetcher{} - converter = tend_mocks.MockTendConverter{} - - transformer = tend.TendTransformer{ - Repository: &repository, + headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + transformer = factories.Transformer{ + Config: config, Fetcher: &fetcher, Converter: &converter, - Config: tend.TendConfig, - } + Repository: &repository, + }.NewTransformer(nil, nil) }) It("gets missing headers for blocks in the configured range", func() { @@ -68,12 +72,12 @@ var _ = Describe("Tend Transformer", func() { }) It("fetches eth logs for each missing header", func() { - repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1}, {BlockNumber: blockNumber2}}) + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) expectedTopics := [][]common.Hash{{common.HexToHash(shared.TendFunctionSignature)}} err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{blockNumber1, blockNumber2})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{tend.TendConfig.ContractAddresses, tend.TendConfig.ContractAddresses})) }) @@ -88,34 +92,17 @@ var _ = Describe("Tend Transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &tend_mocks.MockTendConverter{} - mockRepository := &tend_mocks.MockTendRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := tend.TendTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &tend_mocks.MockTendConverter{} - mockRepository := &tend_mocks.MockTendRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := tend.TendTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedErr(fakes.FakeError) err := transformer.Execute() @@ -124,37 +111,37 @@ var _ = Describe("Tend Transformer", func() { }) It("converts an eth log to a Model", func() { - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + repository.SetMissingHeaders([]core.Header{headerOne}) fetcher.SetFetchedLogs([]types.Log{test_data.TendLogNote}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(converter.LogsToConvert).To(Equal([]types.Log{test_data.TendLogNote})) + Expect(converter.PassedLogs).To(Equal([]types.Log{test_data.TendLogNote})) }) It("returns an error if converter fails", func() { - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) + repository.SetMissingHeaders([]core.Header{headerOne}) fetcher.SetFetchedLogs([]types.Log{test_data.TendLogNote}) converter.SetConverterError(fakes.FakeError) err := transformer.Execute() + Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("persists the tend record", func() { - headerId := int64(1) - repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1, Id: headerId}}) + repository.SetMissingHeaders([]core.Header{headerOne}) fetcher.SetFetchedLogs([]types.Log{test_data.TendLogNote}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(headerId)) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) Expect(repository.PassedTendModel).To(Equal(test_data.TendModel)) }) It("returns error if persisting tend record fails", func() { - repository.SetMissingHeaders([]core.Header{{BlockNumber: blockNumber1}}) + repository.SetMissingHeaders([]core.Header{headerOne}) fetcher.SetFetchedLogs([]types.Log{test_data.TendLogNote}) repository.SetCreateError(fakes.FakeError) err := transformer.Execute() diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 6cd37253..96aa26ab 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -70,9 +70,15 @@ var ( PitFileIlkTransformerInitializer = ilk.PitFileIlkTransformerInitializer{Config: pitFileConfig}.NewPitFileIlkTransformer PitFileStabilityFeeTransformerInitializer = stability_fee.PitFileStabilityFeeTransformerInitializer{Config: pitFileConfig}.NewPitFileStabilityFeeTransformer PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer - TendTransformerInitializer = tend.TendTransformerInitializer{Config: tend.TendConfig}.NewTendTransformer VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer - VatInitTransformerInitializer = factories.Transformer{ + TendTransformerInitializer = factories.Transformer{ + Config: tend.TendConfig, + Converter: &tend.TendConverter{}, + Repository: &tend.TendRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + + VatInitTransformerInitializer = factories.Transformer{ Config: vat_init.VatInitConfig, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, From 57b3744085d60cb6d151841ed485d8f75ecb4f54 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 12 Oct 2018 16:18:00 +0200 Subject: [PATCH 13/36] Update tend tests --- .../test_data/mocks/tend/converter.go | 13 ++++++------ .../test_data/mocks/tend/repository.go | 20 ++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pkg/transformers/test_data/mocks/tend/converter.go b/pkg/transformers/test_data/mocks/tend/converter.go index 77ec60c9..2669e965 100644 --- a/pkg/transformers/test_data/mocks/tend/converter.go +++ b/pkg/transformers/test_data/mocks/tend/converter.go @@ -17,20 +17,19 @@ package tend import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) type MockTendConverter struct { - LogsToConvert []types.Log + PassedLogs []types.Log ConverterError error } -func (c *MockTendConverter) ToModels(ethLogs []types.Log) ([]tend.TendModel, error) { - c.LogsToConvert = append(c.LogsToConvert, ethLogs...) - return []tend.TendModel{test_data.TendModel}, c.ConverterError +func (converter *MockTendConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + converter.PassedLogs = ethLogs + return []interface{}{test_data.TendModel}, converter.ConverterError } -func (c *MockTendConverter) SetConverterError(err error) { - c.ConverterError = err +func (converter *MockTendConverter) SetConverterError(err error) { + converter.ConverterError = err } diff --git a/pkg/transformers/test_data/mocks/tend/repository.go b/pkg/transformers/test_data/mocks/tend/repository.go index 6bb7da13..cfb6e9af 100644 --- a/pkg/transformers/test_data/mocks/tend/repository.go +++ b/pkg/transformers/test_data/mocks/tend/repository.go @@ -16,9 +16,9 @@ package tend import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" ) type MockTendRepository struct { @@ -26,14 +26,14 @@ type MockTendRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedStartingBlockNumber int64 - PassedTendModel tend.TendModel - markHeaderCheckedErr error + PassedTendModel interface{} + markHeaderCheckedError error markHeaderCheckedPassedHeaderId int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error } -func (repository *MockTendRepository) Create(headerId int64, tend []tend.TendModel) error { +func (repository *MockTendRepository) Create(headerId int64, tend []interface{}) error { repository.PassedHeaderID = headerId repository.PassedTendModel = tend[0] return repository.createError @@ -44,11 +44,11 @@ func (repository *MockTendRepository) SetCreateError(err error) { } func (repository *MockTendRepository) SetMarkHeaderCheckedErr(err error) { - repository.markHeaderCheckedErr = err + repository.markHeaderCheckedError = err } func (repository *MockTendRepository) SetMissingHeadersErr(err error) { - repository.missingHeadersErr = err + repository.missingHeadersError = err } func (repository *MockTendRepository) SetMissingHeaders(headers []core.Header) { @@ -57,15 +57,17 @@ func (repository *MockTendRepository) SetMissingHeaders(headers []core.Header) { func (repository *MockTendRepository) MarkHeaderChecked(headerId int64) error { repository.markHeaderCheckedPassedHeaderId = headerId - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockTendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } func (repository *MockTendRepository) AssertMarkHeaderCheckedCalledWith(headerId int64) { Expect(repository.markHeaderCheckedPassedHeaderId).To(Equal(headerId)) } + +func (repository *MockTendRepository) SetDB(db *postgres.DB) {} From 2cc303a2f5679f8b377ba9688736c8e1a68512a8 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 11:35:05 +0200 Subject: [PATCH 14/36] Refactor stability_fee transfomer --- .../pit_file/stability_fee/config.go | 28 +++++++ .../pit_file/stability_fee/converter.go | 8 +- .../pit_file/stability_fee/repository.go | 37 +++++---- .../pit_file/stability_fee/transformer.go | 79 ------------------- pkg/transformers/transformers.go | 10 ++- 5 files changed, 56 insertions(+), 106 deletions(-) create mode 100644 pkg/transformers/pit_file/stability_fee/config.go delete mode 100644 pkg/transformers/pit_file/stability_fee/transformer.go diff --git a/pkg/transformers/pit_file/stability_fee/config.go b/pkg/transformers/pit_file/stability_fee/config.go new file mode 100644 index 00000000..5c6e4e07 --- /dev/null +++ b/pkg/transformers/pit_file/stability_fee/config.go @@ -0,0 +1,28 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stability_fee + +import ( + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var StabilityFeeFileConfig = shared.TransformerConfig{ + TransformerName: "stability fee", + ContractAddresses: []string{shared.PitContractAddress}, + ContractAbi: shared.PitABI, + Topics: []string{shared.PitFileStabilityFeeSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/pit_file/stability_fee/converter.go b/pkg/transformers/pit_file/stability_fee/converter.go index 27d20155..6f47ec51 100644 --- a/pkg/transformers/pit_file/stability_fee/converter.go +++ b/pkg/transformers/pit_file/stability_fee/converter.go @@ -23,14 +23,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error) -} - type PitFileStabilityFeeConverter struct{} -func (PitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]PitFileStabilityFeeModel, error) { - var models []PitFileStabilityFeeModel +func (PitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { diff --git a/pkg/transformers/pit_file/stability_fee/repository.go b/pkg/transformers/pit_file/stability_fee/repository.go index 6942512f..e94b6a5e 100644 --- a/pkg/transformers/pit_file/stability_fee/repository.go +++ b/pkg/transformers/pit_file/stability_fee/repository.go @@ -19,51 +19,46 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []PitFileStabilityFeeModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type PitFileStabilityFeeRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewPitFileStabilityFeeRepository(db *postgres.DB) PitFileStabilityFeeRepository { - return PitFileStabilityFeeRepository{ - db: db, - } -} - -func (repository PitFileStabilityFeeRepository) Create(headerID int64, models []PitFileStabilityFeeModel) error { - tx, err := repository.db.Begin() +func (repository PitFileStabilityFeeRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + + var pitFileSF PitFileStabilityFeeModel for _, model := range models { + pitFileSF = model.(PitFileStabilityFeeModel) _, err = tx.Exec( `INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5)`, - headerID, model.What, model.Data, model.TransactionIndex, model.Raw, + headerID, pitFileSF.What, pitFileSF.Data, pitFileSF.TransactionIndex, pitFileSF.Raw, ) + if err != nil { tx.Rollback() return err } } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true) + if err != nil { tx.Rollback() return err } + return tx.Commit() } func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true) @@ -72,7 +67,7 @@ func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64 func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -82,7 +77,11 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *PitFileStabilityFeeRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/pit_file/stability_fee/transformer.go b/pkg/transformers/pit_file/stability_fee/transformer.go deleted file mode 100644 index 781c223e..00000000 --- a/pkg/transformers/pit_file/stability_fee/transformer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package stability_fee - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type PitFileStabilityFeeTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -type PitFileStabilityFeeTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer PitFileStabilityFeeTransformerInitializer) NewPitFileStabilityFeeTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := PitFileStabilityFeeConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewPitFileStabilityFeeRepository(db) - return PitFileStabilityFeeTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -func (transformer PitFileStabilityFeeTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching pit file stability fee event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.PitFileStabilityFeeSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(pit_file.PitFileConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - models, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, models) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 96aa26ab..0499b4ab 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -68,10 +68,16 @@ var ( pitFileConfig = pit_file.PitFileConfig PitFileDebtCeilingTransformerInitializer = debt_ceiling.PitFileDebtCeilingTransformerInitializer{Config: pitFileConfig}.NewPitFileDebtCeilingTransformer PitFileIlkTransformerInitializer = ilk.PitFileIlkTransformerInitializer{Config: pitFileConfig}.NewPitFileIlkTransformer - PitFileStabilityFeeTransformerInitializer = stability_fee.PitFileStabilityFeeTransformerInitializer{Config: pitFileConfig}.NewPitFileStabilityFeeTransformer PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer - TendTransformerInitializer = factories.Transformer{ + PitFileStabilityFeeTransformerInitializer = factories.Transformer{ + Config: stability_fee.StabilityFeeFileConfig, + Converter: &stability_fee.PitFileStabilityFeeConverter{}, + Repository: &stability_fee.PitFileStabilityFeeRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + + TendTransformerInitializer = factories.Transformer{ Config: tend.TendConfig, Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, From 34128661822356818bc101d68a2e1594b403cb30 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 11:38:07 +0200 Subject: [PATCH 15/36] Update tests for stability_fee --- .../pit_file/stability_fee/repository_test.go | 39 ++--- .../stability_fee/transformer_test.go | 136 ++++++------------ .../mocks/pit_file/stability_fee/converter.go | 11 +- .../pit_file/stability_fee/repository.go | 26 ++-- 4 files changed, 82 insertions(+), 130 deletions(-) diff --git a/pkg/transformers/pit_file/stability_fee/repository_test.go b/pkg/transformers/pit_file/stability_fee/repository_test.go index 499f71fa..62f3b416 100644 --- a/pkg/transformers/pit_file/stability_fee/repository_test.go +++ b/pkg/transformers/pit_file/stability_fee/repository_test.go @@ -32,27 +32,31 @@ import ( var _ = Describe("Pit file stability fee repository", func() { var ( - db *postgres.DB - pitFileRepository stability_fee.Repository - err error - headerRepository datastore.HeaderRepository + db *postgres.DB + pitFileStabilityFeeRepository stability_fee.PitFileStabilityFeeRepository + err error + headerRepository datastore.HeaderRepository ) BeforeEach(func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - pitFileRepository = stability_fee.NewPitFileStabilityFeeRepository(db) + pitFileStabilityFeeRepository = stability_fee.PitFileStabilityFeeRepository{DB: db} }) Describe("Create", func() { - var headerID int64 + var ( + db *postgres.DB + err error + headerID int64 + ) BeforeEach(func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = pitFileRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) + err = pitFileStabilityFeeRepository.Create(headerID, []interface{}{test_data.PitFileStabilityFeeModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -74,7 +78,7 @@ var _ = Describe("Pit file stability fee repository", func() { }) It("does not duplicate pit file events", func() { - err = pitFileRepository.Create(headerID, []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}) + err = pitFileStabilityFeeRepository.Create(headerID, []interface{}{test_data.PitFileStabilityFeeModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -100,7 +104,7 @@ var _ = Describe("Pit file stability fee repository", func() { }) It("creates a row for a new headerID", func() { - err = pitFileRepository.MarkHeaderChecked(headerID) + err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID) Expect(err).NotTo(HaveOccurred()) var headerChecked bool @@ -112,7 +116,7 @@ var _ = Describe("Pit file stability fee repository", func() { It("updates row when headerID already exists", func() { _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) - err = pitFileRepository.MarkHeaderChecked(headerID) + err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerID) Expect(err).NotTo(HaveOccurred()) var headerChecked bool @@ -144,10 +148,10 @@ var _ = Describe("Pit file stability fee repository", func() { }) It("returns headers that haven't been checked", func() { - err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) + err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock) + headers, err := pitFileStabilityFeeRepository.MissingHeaders(startingBlock, endingBlock) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(2)) @@ -156,10 +160,10 @@ var _ = Describe("Pit file stability fee repository", func() { }) It("only treats headers as checked if pit file stability fee logs have been checked", func() { - _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock) + headers, err := pitFileStabilityFeeRepository.MissingHeaders(startingBlock, endingBlock) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(3)) @@ -175,11 +179,12 @@ var _ = Describe("Pit file stability fee repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - pitFileRepositoryTwo := stability_fee.NewPitFileStabilityFeeRepository(dbTwo) - err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) + + pitFileRepositoryTwo := stability_fee.PitFileStabilityFeeRepository{DB: dbTwo} + err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeOneMissingHeaders, err := pitFileStabilityFeeRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) diff --git a/pkg/transformers/pit_file/stability_fee/transformer_test.go b/pkg/transformers/pit_file/stability_fee/transformer_test.go index b2464af7..58a6bba4 100644 --- a/pkg/transformers/pit_file/stability_fee/transformer_test.go +++ b/pkg/transformers/pit_file/stability_fee/transformer_test.go @@ -19,11 +19,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" @@ -31,15 +32,31 @@ import ( ) var _ = Describe("Pit file stability fee transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Config: pit_file.PitFileConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &stability_fee_mocks.MockPitFileStabilityFeeConverter{}, - Repository: repository, - } + var ( + config = stability_fee.StabilityFeeFileConfig + fetcher mocks.MockLogFetcher + converter stability_fee_mocks.MockPitFileStabilityFeeConverter + repository stability_fee_mocks.MockPitFileStabilityFeeRepository + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) + BeforeEach(func() { + fetcher = mocks.MockLogFetcher{} + converter = stability_fee_mocks.MockPitFileStabilityFeeConverter{} + repository = stability_fee_mocks.MockPitFileStabilityFeeRepository{} + transformer = factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, + }.NewTransformer(nil, nil) + headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + }) + + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -48,14 +65,7 @@ var _ = Describe("Pit file stability fee transformer", func() { }) It("returns error if repository returns error for missing headers", func() { - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} repository.SetMissingHeadersErr(fakes.FakeError) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &stability_fee_mocks.MockPitFileStabilityFeeConverter{}, - Repository: repository, - } - err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -63,33 +73,19 @@ var _ = Describe("Pit file stability fee transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: fetcher, - Converter: &stability_fee_mocks.MockPitFileStabilityFeeConverter{}, - Repository: repository, - } - + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{pit_file.PitFileConfig.ContractAddresses, pit_file.PitFileConfig.ContractAddresses})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{ + pit_file.PitFileConfig.ContractAddresses, pit_file.PitFileConfig.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileStabilityFeeSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: fetcher, - Converter: &stability_fee_mocks.MockPitFileStabilityFeeConverter{}, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -98,34 +94,17 @@ var _ = Describe("Pit file stability fee transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} - mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} - mockRepository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedErr(fakes.FakeError) err := transformer.Execute() @@ -134,16 +113,8 @@ var _ = Describe("Pit file stability fee transformer", func() { }) It("converts matching logs", func() { - converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog}) - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -152,17 +123,9 @@ var _ = Describe("Pit file stability fee transformer", func() { }) It("returns error if converter returns error", func() { - converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog}) - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -171,37 +134,20 @@ var _ = Describe("Pit file stability fee transformer", func() { }) It("persists pit file model", func() { - converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog}) - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.PitFileStabilityFeeModel})) }) It("returns error if repository returns error for create", func() { - converter := &stability_fee_mocks.MockPitFileStabilityFeeConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileStabilityFeeLog}) - repository := &stability_fee_mocks.MockPitFileStabilityFeeRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := stability_fee.PitFileStabilityFeeTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() diff --git a/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go b/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go index 8539d367..6e342cad 100644 --- a/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go +++ b/pkg/transformers/test_data/mocks/pit_file/stability_fee/converter.go @@ -17,20 +17,19 @@ package stability_fee import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) type MockPitFileStabilityFeeConverter struct { - converterErr error - PassedLogs []types.Log + converterError error + PassedLogs []types.Log } -func (converter *MockPitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]stability_fee.PitFileStabilityFeeModel, error) { +func (converter *MockPitFileStabilityFeeConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []stability_fee.PitFileStabilityFeeModel{test_data.PitFileStabilityFeeModel}, converter.converterErr + return []interface{}{test_data.PitFileStabilityFeeModel}, converter.converterError } func (converter *MockPitFileStabilityFeeConverter) SetConverterError(e error) { - converter.converterErr = e + converter.converterError = e } diff --git a/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go b/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go index 907538a8..53967bd0 100644 --- a/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go @@ -16,46 +16,46 @@ package stability_fee import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" ) type MockPitFileStabilityFeeRepository struct { - createErr error - markHeaderCheckedErr error + createError error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []stability_fee.PitFileStabilityFeeModel + PassedModels []interface{} } -func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, models []stability_fee.PitFileStabilityFeeModel) error { +func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, models []interface{}) error { repository.PassedModels = models repository.PassedHeaderID = headerID - return repository.createErr + return repository.createError } func (repository *MockPitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockPitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } func (repository *MockPitFileStabilityFeeRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e + repository.markHeaderCheckedError = e } func (repository *MockPitFileStabilityFeeRepository) SetMissingHeadersErr(e error) { - repository.missingHeadersErr = e + repository.missingHeadersError = e } func (repository *MockPitFileStabilityFeeRepository) SetMissingHeaders(headers []core.Header) { @@ -63,9 +63,11 @@ func (repository *MockPitFileStabilityFeeRepository) SetMissingHeaders(headers [ } func (repository *MockPitFileStabilityFeeRepository) SetCreateError(e error) { - repository.createErr = e + repository.createError = e } func (repository *MockPitFileStabilityFeeRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockPitFileStabilityFeeRepository) SetDB(db *postgres.DB) {} From 837a777c8a094fbec8a74b2121d6c869cc14aaf5 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 11:39:05 +0200 Subject: [PATCH 16/36] Update transformer name for stability fee --- pkg/transformers/pit_file/stability_fee/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/transformers/pit_file/stability_fee/config.go b/pkg/transformers/pit_file/stability_fee/config.go index 5c6e4e07..1ca34940 100644 --- a/pkg/transformers/pit_file/stability_fee/config.go +++ b/pkg/transformers/pit_file/stability_fee/config.go @@ -19,7 +19,7 @@ import ( ) var StabilityFeeFileConfig = shared.TransformerConfig{ - TransformerName: "stability fee", + TransformerName: "pit file stability fee", ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, Topics: []string{shared.PitFileStabilityFeeSignature}, From 98062a79708d6b9d8ef4963ea2d0d449e736829a Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 11:53:13 +0200 Subject: [PATCH 17/36] Refactor pit file ilk transfomer --- pkg/transformers/pit_file/ilk/config.go | 28 +++++++ pkg/transformers/pit_file/ilk/converter.go | 8 +- pkg/transformers/pit_file/ilk/repository.go | 41 +++++----- pkg/transformers/pit_file/ilk/transformer.go | 79 -------------------- pkg/transformers/transformers.go | 8 +- 5 files changed, 55 insertions(+), 109 deletions(-) create mode 100644 pkg/transformers/pit_file/ilk/config.go delete mode 100644 pkg/transformers/pit_file/ilk/transformer.go diff --git a/pkg/transformers/pit_file/ilk/config.go b/pkg/transformers/pit_file/ilk/config.go new file mode 100644 index 00000000..19d65719 --- /dev/null +++ b/pkg/transformers/pit_file/ilk/config.go @@ -0,0 +1,28 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ilk + +import ( + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var IlkFileConfig = shared.TransformerConfig{ + TransformerName: "pit file ilk", + ContractAddresses: []string{shared.PitContractAddress}, + ContractAbi: shared.PitABI, + Topics: []string{shared.PitFileIlkSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/pit_file/ilk/converter.go b/pkg/transformers/pit_file/ilk/converter.go index 94777525..823d7823 100644 --- a/pkg/transformers/pit_file/ilk/converter.go +++ b/pkg/transformers/pit_file/ilk/converter.go @@ -24,14 +24,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error) -} - type PitFileIlkConverter struct{} -func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]PitFileIlkModel, error) { - var models []PitFileIlkModel +func (PitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 68ac396b..c6b11c8c 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -19,32 +19,23 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []PitFileIlkModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type PitFileIlkRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewPitFileIlkRepository(db *postgres.DB) PitFileIlkRepository { - return PitFileIlkRepository{ - db: db, - } -} - -func (repository PitFileIlkRepository) Create(headerID int64, models []PitFileIlkModel) error { - tx, err := repository.db.Begin() +func (repository PitFileIlkRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + + var pitFileIlk PitFileIlkModel for _, model := range models { + pitFileIlk = model.(PitFileIlkModel) _, err = tx.Exec( `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, - headerID, model.Ilk, model.What, model.Data, model.TransactionIndex, model.Raw, + headerID, pitFileIlk.Ilk, pitFileIlk.What, pitFileIlk.Data, pitFileIlk.TransactionIndex, pitFileIlk.Raw, ) if err != nil { tx.Rollback() @@ -63,7 +54,7 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []PitFileIl } func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET pit_file_ilk_checked = $2`, headerID, true) @@ -72,17 +63,21 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id + LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR pit_file_ilk_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository PitFileIlkRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/pit_file/ilk/transformer.go b/pkg/transformers/pit_file/ilk/transformer.go deleted file mode 100644 index 2318dca0..00000000 --- a/pkg/transformers/pit_file/ilk/transformer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ilk - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type PitFileIlkTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -type PitFileIlkTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer PitFileIlkTransformerInitializer) NewPitFileIlkTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := PitFileIlkConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewPitFileIlkRepository(db) - return PitFileIlkTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -func (transformer PitFileIlkTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching pit file ilk event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.PitFileIlkSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(pit_file.PitFileConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - models, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, models) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 0499b4ab..dc181fdb 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -67,9 +67,15 @@ var ( FrobTransformerInitializer = frob.FrobTransformerInitializer{Config: frob.FrobConfig}.NewFrobTransformer pitFileConfig = pit_file.PitFileConfig PitFileDebtCeilingTransformerInitializer = debt_ceiling.PitFileDebtCeilingTransformerInitializer{Config: pitFileConfig}.NewPitFileDebtCeilingTransformer - PitFileIlkTransformerInitializer = ilk.PitFileIlkTransformerInitializer{Config: pitFileConfig}.NewPitFileIlkTransformer PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer + PitFileIlkTransformerInitializer = factories.Transformer{ + Config: ilk.IlkFileConfig, + Converter: &ilk.PitFileIlkConverter{}, + Repository: &ilk.PitFileIlkRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + PitFileStabilityFeeTransformerInitializer = factories.Transformer{ Config: stability_fee.StabilityFeeFileConfig, Converter: &stability_fee.PitFileStabilityFeeConverter{}, From 399b3b78a09874d65e158ee9b1e063bc8762805e Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 12:46:00 +0200 Subject: [PATCH 18/36] Update tests for pit file ilk transformer --- .../pit_file/ilk/converter_test.go | 2 +- .../pit_file/ilk/repository_test.go | 19 ++- .../pit_file/ilk/transformer_test.go | 131 ++++++------------ .../test_data/mocks/pit_file/ilk/converter.go | 5 +- .../mocks/pit_file/ilk/repository.go | 24 ++-- 5 files changed, 68 insertions(+), 113 deletions(-) diff --git a/pkg/transformers/pit_file/ilk/converter_test.go b/pkg/transformers/pit_file/ilk/converter_test.go index 6dba9b9b..8ddfa4e4 100644 --- a/pkg/transformers/pit_file/ilk/converter_test.go +++ b/pkg/transformers/pit_file/ilk/converter_test.go @@ -54,6 +54,6 @@ var _ = Describe("Pit file ilk converter", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(models)).To(Equal(1)) - Expect(models[0]).To(Equal(test_data.PitFileIlkModel)) + Expect(models[0].(ilk.PitFileIlkModel)).To(Equal(test_data.PitFileIlkModel)) }) }) diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index feaf9dcd..1c6a7225 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -33,7 +33,7 @@ import ( var _ = Describe("Pit file ilk repository", func() { var ( db *postgres.DB - pitFileRepository ilk.Repository + pitFileRepository ilk.PitFileIlkRepository err error headerRepository datastore.HeaderRepository ) @@ -42,18 +42,24 @@ var _ = Describe("Pit file ilk repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - pitFileRepository = ilk.NewPitFileIlkRepository(db) + pitFileRepository = ilk.PitFileIlkRepository{DB: db} }) Describe("Create", func() { - var headerID int64 + var ( + db *postgres.DB + pitFileIlkRepository ilk.PitFileIlkRepository + err error + headerID int64 + ) BeforeEach(func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = pitFileRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) + err = pitFileRepository.Create(headerID, []interface{}{test_data.PitFileIlkModel}) Expect(err).NotTo(HaveOccurred()) + pitFileIlkRepository = ilk.PitFileIlkRepository{DB: db} }) It("adds a pit file ilk event", func() { @@ -75,7 +81,7 @@ var _ = Describe("Pit file ilk repository", func() { }) It("does not duplicate pit file ilk events", func() { - err = pitFileRepository.Create(headerID, []ilk.PitFileIlkModel{test_data.PitFileIlkModel}) + err = pitFileRepository.Create(headerID, []interface{}{test_data.PitFileIlkModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -176,7 +182,8 @@ var _ = Describe("Pit file ilk repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - pitFileRepositoryTwo := ilk.NewPitFileIlkRepository(dbTwo) + + pitFileRepositoryTwo := ilk.PitFileIlkRepository{DB: dbTwo} err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/pit_file/ilk/transformer_test.go b/pkg/transformers/pit_file/ilk/transformer_test.go index 2b6cf806..6f030a92 100644 --- a/pkg/transformers/pit_file/ilk/transformer_test.go +++ b/pkg/transformers/pit_file/ilk/transformer_test.go @@ -19,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -31,15 +32,31 @@ import ( ) var _ = Describe("Pit file ilk transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - transformer := ilk.PitFileIlkTransformer{ - Config: pit_file.PitFileConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &pit_file_ilk_mocks.MockPitFileIlkConverter{}, - Repository: repository, - } + var ( + config = ilk.IlkFileConfig + fetcher mocks.MockLogFetcher + converter pit_file_ilk_mocks.MockPitFileIlkConverter + repository pit_file_ilk_mocks.MockPitFileIlkRepository + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) + BeforeEach(func() { + fetcher = mocks.MockLogFetcher{} + converter = pit_file_ilk_mocks.MockPitFileIlkConverter{} + repository = pit_file_ilk_mocks.MockPitFileIlkRepository{} + headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + transformer = factories.Transformer{ + Config: config, + Converter: &converter, + Repository: &repository, + Fetcher: &fetcher, + }.NewTransformer(nil, nil) + }) + + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -48,13 +65,7 @@ var _ = Describe("Pit file ilk transformer", func() { }) It("returns error if repository returns error for missing headers", func() { - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - repository.SetMissingHeadersErr(fakes.FakeError) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &pit_file_ilk_mocks.MockPitFileIlkConverter{}, - Repository: repository, - } + repository.SetMissingHeadersError(fakes.FakeError) err := transformer.Execute() @@ -63,33 +74,19 @@ var _ = Describe("Pit file ilk transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: fetcher, - Converter: &pit_file_ilk_mocks.MockPitFileIlkConverter{}, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{pit_file.PitFileConfig.ContractAddresses, pit_file.PitFileConfig.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileIlkSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: fetcher, - Converter: &pit_file_ilk_mocks.MockPitFileIlkConverter{}, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -98,34 +95,17 @@ var _ = Describe("Pit file ilk transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} - mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := ilk.PitFileIlkTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} - mockRepository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := ilk.PitFileIlkTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedError(fakes.FakeError) err := transformer.Execute() @@ -134,16 +114,8 @@ var _ = Describe("Pit file ilk transformer", func() { }) It("converts matching logs", func() { - converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog}) - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -152,17 +124,9 @@ var _ = Describe("Pit file ilk transformer", func() { }) It("returns error if converter returns error", func() { - converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog}) - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -171,37 +135,20 @@ var _ = Describe("Pit file ilk transformer", func() { }) It("persists pit file model", func() { - converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog}) - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]ilk.PitFileIlkModel{test_data.PitFileIlkModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.PitFileIlkModel})) }) It("returns error if repository returns error for create", func() { - converter := &pit_file_ilk_mocks.MockPitFileIlkConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileIlkLog}) - repository := &pit_file_ilk_mocks.MockPitFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := ilk.PitFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() diff --git a/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go b/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go index d7052c42..0924b5e9 100644 --- a/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go +++ b/pkg/transformers/test_data/mocks/pit_file/ilk/converter.go @@ -17,7 +17,6 @@ package ilk import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) @@ -30,7 +29,7 @@ func (converter *MockPitFileIlkConverter) SetConverterError(err error) { converter.converterError = err } -func (converter *MockPitFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.PitFileIlkModel, error) { +func (converter *MockPitFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []ilk.PitFileIlkModel{test_data.PitFileIlkModel}, converter.converterError + return []interface{}{test_data.PitFileIlkModel}, converter.converterError } diff --git a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go index 4bb8db36..899ffab4 100644 --- a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go @@ -16,40 +16,40 @@ package ilk import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" ) type MockPitFileIlkRepository struct { createError error PassedEndingBlockNumber int64 - PassedModels []ilk.PitFileIlkModel + PassedModels []interface{} PassedHeaderID int64 PassedStartingBlockNumber int64 - markHeaderCheckedErr error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error } func (repository *MockPitFileIlkRepository) SetCreateError(err error) { repository.createError = err } -func (repository *MockPitFileIlkRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e +func (repository *MockPitFileIlkRepository) SetMarkHeaderCheckedError(e error) { + repository.markHeaderCheckedError = e } -func (repository *MockPitFileIlkRepository) SetMissingHeadersErr(err error) { - repository.missingHeadersErr = err +func (repository *MockPitFileIlkRepository) SetMissingHeadersError(err error) { + repository.missingHeadersError = err } func (repository *MockPitFileIlkRepository) SetMissingHeaders(headers []core.Header) { repository.missingHeaders = headers } -func (repository *MockPitFileIlkRepository) Create(headerID int64, models []ilk.PitFileIlkModel) error { +func (repository *MockPitFileIlkRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models return repository.createError @@ -57,15 +57,17 @@ func (repository *MockPitFileIlkRepository) Create(headerID int64, models []ilk. func (repository *MockPitFileIlkRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockPitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } func (repository *MockPitFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockPitFileIlkRepository) SetDB(db *postgres.DB) {} \ No newline at end of file From 0239c4daf63a33252270436c4a4104534d137a6e Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 13:30:25 +0200 Subject: [PATCH 19/36] Fix sneaky missing asterisk --- pkg/transformers/pit_file/ilk/repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index c6b11c8c..7dc46138 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -78,6 +78,6 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin return result, err } -func (repository PitFileIlkRepository) SetDB(db *postgres.DB) { +func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { repository.DB = db } From 24199cb67eeeffcce507eb03275912170c3c6bf3 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 16 Oct 2018 16:55:24 +0200 Subject: [PATCH 20/36] Refactor pit file debt ceiling transformer --- .../pit_file/{ => debt_ceiling}/config.go | 7 +- .../pit_file/debt_ceiling/converter.go | 8 +- .../pit_file/debt_ceiling/repository.go | 53 ++++++------- .../pit_file/debt_ceiling/transformer.go | 79 ------------------- pkg/transformers/transformers.go | 10 ++- 5 files changed, 39 insertions(+), 118 deletions(-) rename pkg/transformers/pit_file/{ => debt_ceiling}/config.go (80%) delete mode 100644 pkg/transformers/pit_file/debt_ceiling/transformer.go diff --git a/pkg/transformers/pit_file/config.go b/pkg/transformers/pit_file/debt_ceiling/config.go similarity index 80% rename from pkg/transformers/pit_file/config.go rename to pkg/transformers/pit_file/debt_ceiling/config.go index 73656bd4..23ee22b4 100644 --- a/pkg/transformers/pit_file/config.go +++ b/pkg/transformers/pit_file/debt_ceiling/config.go @@ -12,16 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pit_file +package debt_ceiling import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -var PitFileConfig = shared.TransformerConfig{ +var DebtCeilingFileConfig = shared.TransformerConfig{ + TransformerName: "pit file debt ceiling", ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, - Topics: []string{shared.PitFileIlkSignature, shared.PitFileDebtCeilingSignature, shared.PitFileStabilityFeeSignature}, + Topics: []string{shared.PitFileDebtCeilingSignature}, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/pit_file/debt_ceiling/converter.go b/pkg/transformers/pit_file/debt_ceiling/converter.go index f47044f3..620644a8 100644 --- a/pkg/transformers/pit_file/debt_ceiling/converter.go +++ b/pkg/transformers/pit_file/debt_ceiling/converter.go @@ -24,14 +24,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) -} - type PitFileDebtCeilingConverter struct{} -func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]PitFileDebtCeilingModel, error) { - var models []PitFileDebtCeilingModel +func (PitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index a5e86335..831ce12c 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -19,70 +19,69 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []PitFileDebtCeilingModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type PitFileDebtCeilingRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewPitFileDebtCeilingRepository(db *postgres.DB) PitFileDebtCeilingRepository { - return PitFileDebtCeilingRepository{ - db: db, - } -} - -func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []PitFileDebtCeilingModel) error { - tx, err := repository.db.Begin() +func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + + var pitFileDC PitFileDebtCeilingModel for _, model := range models { + pitFileDC = model.(PitFileDebtCeilingModel) _, err = tx.Exec( `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5)`, - headerID, model.What, model.Data, model.TransactionIndex, model.Raw, + headerID, pitFileDC.What, pitFileDC.Data, pitFileDC.TransactionIndex, pitFileDC.Raw, ) + if err != nil { tx.Rollback() return err } } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) + if err != nil { tx.Rollback() return err } + return tx.Commit() } func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) return err } func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id + LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR pit_file_debt_ceiling_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/pit_file/debt_ceiling/transformer.go b/pkg/transformers/pit_file/debt_ceiling/transformer.go deleted file mode 100644 index a82b2eff..00000000 --- a/pkg/transformers/pit_file/debt_ceiling/transformer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package debt_ceiling - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type PitFileDebtCeilingTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -type PitFileDebtCeilingTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer PitFileDebtCeilingTransformerInitializer) NewPitFileDebtCeilingTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := PitFileDebtCeilingConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewPitFileDebtCeilingRepository(db) - return PitFileDebtCeilingTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -func (transformer PitFileDebtCeilingTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching pit file debt ceiling event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.PitFileDebtCeilingSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(pit_file.PitFileConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - model, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index dc181fdb..1612535e 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -31,7 +31,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/flip_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/flop_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/frob" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" @@ -65,10 +64,15 @@ var ( FlipKickTransformerInitializer = flip_kick.FlipKickTransformerInitializer{Config: flip_kick.FlipKickConfig}.NewFlipKickTransformer FlopKickTransformerInitializer = flop_kick.FlopKickTransformerInitializer{Config: flop_kick.Config}.NewFlopKickTransformer FrobTransformerInitializer = frob.FrobTransformerInitializer{Config: frob.FrobConfig}.NewFrobTransformer - pitFileConfig = pit_file.PitFileConfig - PitFileDebtCeilingTransformerInitializer = debt_ceiling.PitFileDebtCeilingTransformerInitializer{Config: pitFileConfig}.NewPitFileDebtCeilingTransformer PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer + PitFileDebtCeilingTransformerInitializer = factories.Transformer{ + Config: debt_ceiling.DebtCeilingFileConfig, + Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, + Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + PitFileIlkTransformerInitializer = factories.Transformer{ Config: ilk.IlkFileConfig, Converter: &ilk.PitFileIlkConverter{}, From d9b6603109a4260f728f4ae398e1541dd74a8d1e Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 17 Oct 2018 12:33:52 +0200 Subject: [PATCH 21/36] Update pit file debt ceiling tests --- .../pit_file/debt_ceiling/converter_test.go | 4 +- .../pit_file/debt_ceiling/repository_test.go | 29 ++-- .../pit_file/debt_ceiling/transformer_test.go | 141 ++++++------------ .../mocks/pit_file/debt_ceiling/converter.go | 11 +- .../mocks/pit_file/debt_ceiling/repository.go | 30 ++-- 5 files changed, 80 insertions(+), 135 deletions(-) diff --git a/pkg/transformers/pit_file/debt_ceiling/converter_test.go b/pkg/transformers/pit_file/debt_ceiling/converter_test.go index fec50524..2ed70a59 100644 --- a/pkg/transformers/pit_file/debt_ceiling/converter_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/converter_test.go @@ -17,10 +17,10 @@ package debt_ceiling_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) @@ -54,6 +54,6 @@ var _ = Describe("Pit file debt ceiling converter", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(models)).To(Equal(1)) - Expect(models[0]).To(Equal(test_data.PitFileDebtCeilingModel)) + Expect(models[0].(debt_ceiling.PitFileDebtCeilingModel)).To(Equal(test_data.PitFileDebtCeilingModel)) }) }) diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index dd2d729a..ab19f0a2 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -32,17 +32,17 @@ import ( var _ = Describe("Pit file debt ceiling repository", func() { var ( - db *postgres.DB - pitFileRepository debt_ceiling.Repository - err error - headerRepository datastore.HeaderRepository + db *postgres.DB + pitFileDebtCeilingRepository debt_ceiling.PitFileDebtCeilingRepository + err error + headerRepository datastore.HeaderRepository ) BeforeEach(func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - pitFileRepository = debt_ceiling.NewPitFileDebtCeilingRepository(db) + pitFileDebtCeilingRepository = debt_ceiling.PitFileDebtCeilingRepository{DB: db} }) Describe("Create", func() { @@ -52,7 +52,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = pitFileRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) + err = pitFileDebtCeilingRepository.Create(headerID, []interface{}{test_data.PitFileDebtCeilingModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -74,7 +74,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { }) It("does not duplicate pit file events", func() { - err = pitFileRepository.Create(headerID, []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}) + err = pitFileDebtCeilingRepository.Create(headerID, []interface{}{test_data.PitFileDebtCeilingModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -100,7 +100,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { }) It("creates a row for a new headerID", func() { - err = pitFileRepository.MarkHeaderChecked(headerID) + err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID) Expect(err).NotTo(HaveOccurred()) var headerChecked bool @@ -112,7 +112,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { It("updates row when headerID already exists", func() { _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerID) - err = pitFileRepository.MarkHeaderChecked(headerID) + err = pitFileDebtCeilingRepository.MarkHeaderChecked(headerID) Expect(err).NotTo(HaveOccurred()) var headerChecked bool @@ -144,10 +144,11 @@ var _ = Describe("Pit file debt ceiling repository", func() { }) It("returns headers that haven't been checked", func() { - err := pitFileRepository.MarkHeaderChecked(headerIDs[1]) + + err := pitFileDebtCeilingRepository.MarkHeaderChecked(headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock) + headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlock, endingBlock) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(2)) @@ -159,7 +160,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := pitFileRepository.MissingHeaders(startingBlock, endingBlock) + headers, err := pitFileDebtCeilingRepository.MissingHeaders(startingBlock, endingBlock) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(3)) @@ -175,8 +176,8 @@ var _ = Describe("Pit file debt ceiling repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - pitFileRepository := debt_ceiling.NewPitFileDebtCeilingRepository(db) - pitFileRepositoryTwo := debt_ceiling.NewPitFileDebtCeilingRepository(dbTwo) + pitFileRepository := debt_ceiling.PitFileDebtCeilingRepository{DB: db} + pitFileRepositoryTwo := debt_ceiling.PitFileDebtCeilingRepository{DB: dbTwo} err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go index 89dc5b67..96c9d9d1 100644 --- a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go @@ -19,10 +19,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -31,31 +31,40 @@ import ( ) var _ = Describe("Pit file debt ceiling transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Config: pit_file.PitFileConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}, - Repository: repository, - } + var ( + config = debt_ceiling.DebtCeilingFileConfig + fetcher mocks.MockLogFetcher + converter debt_ceiling_mocks.MockPitFileDebtCeilingConverter + repository debt_ceiling_mocks.MockPitFileDebtCeilingRepository + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) + BeforeEach(func() { + fetcher = mocks.MockLogFetcher{} + converter = debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} + repository = debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} + headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + transformer = factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, + } + }) + + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber)) + Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) }) It("returns error if repository returns error for missing headers", func() { - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - repository.SetMissingHeadersErr(fakes.FakeError) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}, - Repository: repository, - } - + repository.SetMissingHeadersError(fakes.FakeError) err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -63,34 +72,18 @@ var _ = Describe("Pit file debt ceiling transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: fetcher, - Converter: &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}, - Repository: repository, - } - + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{pit_file.PitFileConfig.ContractAddresses, pit_file.PitFileConfig.ContractAddresses})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileDebtCeilingSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: fetcher, - Converter: &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{}, - Repository: repository, - } - + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -98,34 +91,17 @@ var _ = Describe("Pit file debt ceiling transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} - mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} - mockRepository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedError(fakes.FakeError) err := transformer.Execute() @@ -134,16 +110,8 @@ var _ = Describe("Pit file debt ceiling transformer", func() { }) It("converts matching logs", func() { - converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog}) - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -152,17 +120,9 @@ var _ = Describe("Pit file debt ceiling transformer", func() { }) It("returns error if converter returns error", func() { - converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog}) - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -171,37 +131,20 @@ var _ = Describe("Pit file debt ceiling transformer", func() { }) It("persists pit file model", func() { - converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog}) - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.PitFileDebtCeilingModel})) }) It("returns error if repository returns error for create", func() { - converter := &debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthPitFileDebtCeilingLog}) - repository := &debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := debt_ceiling.PitFileDebtCeilingTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() diff --git a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go index f4ca9aa7..83d828b1 100644 --- a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go +++ b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/converter.go @@ -17,20 +17,19 @@ package debt_ceiling import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) type MockPitFileDebtCeilingConverter struct { - converterErr error - PassedLogs []types.Log + converterError error + PassedLogs []types.Log } -func (converter *MockPitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]debt_ceiling.PitFileDebtCeilingModel, error) { +func (converter *MockPitFileDebtCeilingConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []debt_ceiling.PitFileDebtCeilingModel{test_data.PitFileDebtCeilingModel}, converter.converterErr + return []interface{}{test_data.PitFileDebtCeilingModel}, converter.converterError } func (converter *MockPitFileDebtCeilingConverter) SetConverterError(e error) { - converter.converterErr = e + converter.converterError = e } diff --git a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go index be06442e..4ba8ec08 100644 --- a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go @@ -16,46 +16,46 @@ package debt_ceiling import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" ) type MockPitFileDebtCeilingRepository struct { - createErr error - markHeaderCheckedErr error + createError error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []debt_ceiling.PitFileDebtCeilingModel + PassedModels []interface{} } func (repository *MockPitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } -func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, models []debt_ceiling.PitFileDebtCeilingModel) error { +func (repository *MockPitFileDebtCeilingRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models - return repository.createErr + return repository.createError } func (repository *MockPitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } -func (repository *MockPitFileDebtCeilingRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e +func (repository *MockPitFileDebtCeilingRepository) SetMarkHeaderCheckedError(e error) { + repository.markHeaderCheckedError = e } -func (repository *MockPitFileDebtCeilingRepository) SetMissingHeadersErr(e error) { - repository.missingHeadersErr = e +func (repository *MockPitFileDebtCeilingRepository) SetMissingHeadersError(e error) { + repository.missingHeadersError = e } func (repository *MockPitFileDebtCeilingRepository) SetMissingHeaders(headers []core.Header) { @@ -63,9 +63,11 @@ func (repository *MockPitFileDebtCeilingRepository) SetMissingHeaders(headers [] } func (repository *MockPitFileDebtCeilingRepository) SetCreateError(e error) { - repository.createErr = e + repository.createError = e } func (repository *MockPitFileDebtCeilingRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockPitFileDebtCeilingRepository) SetDB(db *postgres.DB) {} From ccca6e0229dfd74935f61cc438a7b08d6d43e944 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 17 Oct 2018 12:34:25 +0200 Subject: [PATCH 22/36] Update old pit file shared config references --- .../pit_file/ilk/transformer_test.go | 23 +++++++++---------- .../stability_fee/transformer_test.go | 7 +++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/transformers/pit_file/ilk/transformer_test.go b/pkg/transformers/pit_file/ilk/transformer_test.go index 6f030a92..89474d55 100644 --- a/pkg/transformers/pit_file/ilk/transformer_test.go +++ b/pkg/transformers/pit_file/ilk/transformer_test.go @@ -23,31 +23,30 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" - pit_file_ilk_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file/ilk" + ilk_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/pit_file/ilk" ) var _ = Describe("Pit file ilk transformer", func() { var ( config = ilk.IlkFileConfig fetcher mocks.MockLogFetcher - converter pit_file_ilk_mocks.MockPitFileIlkConverter - repository pit_file_ilk_mocks.MockPitFileIlkRepository + converter ilk_mocks.MockPitFileIlkConverter + repository ilk_mocks.MockPitFileIlkRepository transformer shared.Transformer headerOne core.Header headerTwo core.Header ) BeforeEach(func() { - fetcher = mocks.MockLogFetcher{} - converter = pit_file_ilk_mocks.MockPitFileIlkConverter{} - repository = pit_file_ilk_mocks.MockPitFileIlkRepository{} - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + fetcher = mocks.MockLogFetcher{} + converter = ilk_mocks.MockPitFileIlkConverter{} + repository = ilk_mocks.MockPitFileIlkRepository{} + headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} transformer = factories.Transformer{ Config: config, Converter: &converter, @@ -60,8 +59,8 @@ var _ = Describe("Pit file ilk transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber)) + Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) }) It("returns error if repository returns error for missing headers", func() { @@ -80,7 +79,7 @@ var _ = Describe("Pit file ilk transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{pit_file.PitFileConfig.ContractAddresses, pit_file.PitFileConfig.ContractAddresses})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileIlkSignature)}})) }) diff --git a/pkg/transformers/pit_file/stability_fee/transformer_test.go b/pkg/transformers/pit_file/stability_fee/transformer_test.go index 58a6bba4..633d7feb 100644 --- a/pkg/transformers/pit_file/stability_fee/transformer_test.go +++ b/pkg/transformers/pit_file/stability_fee/transformer_test.go @@ -24,7 +24,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" @@ -60,8 +59,8 @@ var _ = Describe("Pit file stability fee transformer", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(pit_file.PitFileConfig.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(pit_file.PitFileConfig.EndingBlockNumber)) + Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) }) It("returns error if repository returns error for missing headers", func() { @@ -79,7 +78,7 @@ var _ = Describe("Pit file stability fee transformer", func() { Expect(err).NotTo(HaveOccurred()) Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{ - pit_file.PitFileConfig.ContractAddresses, pit_file.PitFileConfig.ContractAddresses})) + config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.PitFileStabilityFeeSignature)}})) }) From dc4eb009c106f96f83ae2fcf1351f3c5d17afaad Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 18 Oct 2018 11:52:19 +0200 Subject: [PATCH 23/36] Refactor drip_file transformers --- .../drip_file/{ => ilk}/config.go | 7 +- pkg/transformers/drip_file/ilk/converter.go | 6 +- pkg/transformers/drip_file/ilk/repository.go | 51 ++++++------ pkg/transformers/drip_file/ilk/transformer.go | 79 ------------------- pkg/transformers/drip_file/repo/config.go | 26 ++++++ pkg/transformers/drip_file/repo/converter.go | 10 +-- pkg/transformers/drip_file/repo/repository.go | 53 +++++++------ .../drip_file/repo/transformer.go | 79 ------------------- pkg/transformers/drip_file/vow/config.go | 26 ++++++ pkg/transformers/drip_file/vow/converter.go | 6 +- pkg/transformers/drip_file/vow/repository.go | 52 ++++++------ pkg/transformers/drip_file/vow/transformer.go | 79 ------------------- pkg/transformers/transformers.go | 61 ++++++++------ 13 files changed, 184 insertions(+), 351 deletions(-) rename pkg/transformers/drip_file/{ => ilk}/config.go (83%) delete mode 100644 pkg/transformers/drip_file/ilk/transformer.go create mode 100644 pkg/transformers/drip_file/repo/config.go delete mode 100644 pkg/transformers/drip_file/repo/transformer.go create mode 100644 pkg/transformers/drip_file/vow/config.go delete mode 100644 pkg/transformers/drip_file/vow/transformer.go diff --git a/pkg/transformers/drip_file/config.go b/pkg/transformers/drip_file/ilk/config.go similarity index 83% rename from pkg/transformers/drip_file/config.go rename to pkg/transformers/drip_file/ilk/config.go index 7acf5252..9e58fd6e 100644 --- a/pkg/transformers/drip_file/config.go +++ b/pkg/transformers/drip_file/ilk/config.go @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package drip_file +package ilk import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var DripFileConfig = shared.TransformerConfig{ +var DripFileIlkConfig = shared.TransformerConfig{ + TransformerName: "drip file ilk", ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, - Topics: []string{shared.DripFileIlkSignature, shared.DripFileRepoSignature}, + Topics: []string{shared.DripFileIlkSignature}, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/drip_file/ilk/converter.go b/pkg/transformers/drip_file/ilk/converter.go index 7620324e..c2ca2dec 100644 --- a/pkg/transformers/drip_file/ilk/converter.go +++ b/pkg/transformers/drip_file/ilk/converter.go @@ -30,13 +30,14 @@ type Converter interface { type DripFileIlkConverter struct{} -func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error) { - var models []DripFileIlkModel +func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { return nil, err } + ilk := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) vow := string(bytes.Trim(ethLog.Topics[3].Bytes(), "\x00")) taxBytes := ethLog.Data[len(ethLog.Data)-shared.DataItemLength:] @@ -45,6 +46,7 @@ func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]DripFileIlkModel, e if err != nil { return nil, err } + model := DripFileIlkModel{ Ilk: ilk, Vow: vow, diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index d9bad4fe..dc12720b 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -19,31 +19,25 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []DripFileIlkModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type DripFileIlkRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewDripFileIlkRepository(db *postgres.DB) DripFileIlkRepository { - return DripFileIlkRepository{db: db} -} - -func (repository DripFileIlkRepository) Create(headerID int64, models []DripFileIlkModel) error { - tx, err := repository.db.Begin() +func (repository DripFileIlkRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + + var ilk DripFileIlkModel for _, model := range models { + ilk = model.(DripFileIlkModel) _, err = tx.Exec( `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, - headerID, model.Ilk, model.Vow, model.Tax, model.LogIndex, model.TransactionIndex, model.Raw, + VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, + headerID, ilk.Ilk, ilk.Vow, ilk.Tax, ilk.LogIndex, ilk.TransactionIndex, ilk.Raw, ) + if err != nil { tx.Rollback() return err @@ -51,36 +45,41 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []DripFile } _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET drip_file_ilk_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_ilk_checked = $2`, headerID, true) if err != nil { tx.Rollback() return err } + return tx.Commit() } func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET drip_file_ilk_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_ilk_checked = $2`, headerID, true) return err } func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id + LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR drip_file_ilk_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/drip_file/ilk/transformer.go b/pkg/transformers/drip_file/ilk/transformer.go deleted file mode 100644 index d8a518f2..00000000 --- a/pkg/transformers/drip_file/ilk/transformer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ilk - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type DripFileIlkTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -type DripFileIlkTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer DripFileIlkTransformerInitializer) NewDripFileIlkTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := DripFileIlkConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewDripFileIlkRepository(db) - return DripFileIlkTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -func (transformer DripFileIlkTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching drip file ilk event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.DripFileIlkSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(drip_file.DripFileConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - model, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/transformers/drip_file/repo/config.go b/pkg/transformers/drip_file/repo/config.go new file mode 100644 index 00000000..f5c52fe5 --- /dev/null +++ b/pkg/transformers/drip_file/repo/config.go @@ -0,0 +1,26 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var DripFileRepoConfig = shared.TransformerConfig{ + TransformerName: "drip file repo", + ContractAddresses: []string{shared.DripContractAddress}, + ContractAbi: shared.DripABI, + Topics: []string{shared.DripFileRepoSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/drip_file/repo/converter.go b/pkg/transformers/drip_file/repo/converter.go index 8dfb29c6..ba4aa775 100644 --- a/pkg/transformers/drip_file/repo/converter.go +++ b/pkg/transformers/drip_file/repo/converter.go @@ -22,25 +22,23 @@ import ( "math/big" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error) -} - type DripFileRepoConverter struct{} -func (DripFileRepoConverter) ToModels(ethLogs []types.Log) ([]DripFileRepoModel, error) { - var models []DripFileRepoModel +func (DripFileRepoConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { return nil, err } + what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) data := big.NewInt(0).SetBytes(ethLog.Topics[3].Bytes()).String() raw, err := json.Marshal(ethLog) if err != nil { return nil, err } + model := DripFileRepoModel{ What: what, Data: data, diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 3ea04d13..7b28c58f 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -19,68 +19,69 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []DripFileRepoModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type DripFileRepoRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewDripFileRepoRepository(db *postgres.DB) DripFileRepoRepository { - return DripFileRepoRepository{db: db} -} - -func (repository DripFileRepoRepository) Create(headerID int64, models []DripFileRepoModel) error { - tx, err := repository.db.Begin() +func (repository DripFileRepoRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + + var repo DripFileRepoModel for _, model := range models { + repo = model.(DripFileRepoModel) _, err = tx.Exec( `INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, - headerID, model.What, model.Data, model.LogIndex, model.TransactionIndex, model.Raw, + VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, + headerID, repo.What, repo.Data, repo.LogIndex, repo.TransactionIndex, repo.Raw, ) + if err != nil { tx.Rollback() return err } } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET drip_file_repo_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_repo_checked = $2`, headerID, true) + if err != nil { tx.Rollback() return err } + return tx.Commit() } func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET drip_file_repo_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_repo_checked = $2`, headerID, true) return err } func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id + LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR drip_file_repo_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/drip_file/repo/transformer.go b/pkg/transformers/drip_file/repo/transformer.go deleted file mode 100644 index d34593dc..00000000 --- a/pkg/transformers/drip_file/repo/transformer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package repo - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type DripFileRepoTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -type DripFileRepoTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer DripFileRepoTransformerInitializer) NewDripFileRepoTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := DripFileRepoConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewDripFileRepoRepository(db) - return DripFileRepoTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -func (transformer DripFileRepoTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching drip file repo event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.DripFileRepoSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(drip_file.DripFileConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - model, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/transformers/drip_file/vow/config.go b/pkg/transformers/drip_file/vow/config.go new file mode 100644 index 00000000..2d7f9004 --- /dev/null +++ b/pkg/transformers/drip_file/vow/config.go @@ -0,0 +1,26 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vow + +import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + +var DripFileVowConfig = shared.TransformerConfig{ + TransformerName: "drip file vow", + ContractAddresses: []string{shared.DripContractAddress}, + ContractAbi: shared.DripABI, + Topics: []string{shared.DripFileVowSignature}, + StartingBlockNumber: 0, + EndingBlockNumber: 10000000, +} diff --git a/pkg/transformers/drip_file/vow/converter.go b/pkg/transformers/drip_file/vow/converter.go index c299d6ed..a4d48b28 100644 --- a/pkg/transformers/drip_file/vow/converter.go +++ b/pkg/transformers/drip_file/vow/converter.go @@ -28,19 +28,21 @@ type Converter interface { type DripFileVowConverter struct{} -func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]DripFileVowModel, error) { - var models []DripFileVowModel +func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { return nil, err } + what := string(bytes.Trim(ethLog.Topics[2].Bytes(), "\x00")) data := common.BytesToAddress(ethLog.Topics[3].Bytes()).String() raw, err := json.Marshal(ethLog) if err != nil { return nil, err } + model := DripFileVowModel{ What: what, Data: data, diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index bba93b65..4aa39940 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -19,68 +19,68 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository interface { - Create(headerID int64, models []DripFileVowModel) error - MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) -} - type DripFileVowRepository struct { - db *postgres.DB + DB *postgres.DB } -func NewDripFileVowRepository(db *postgres.DB) DripFileVowRepository { - return DripFileVowRepository{db: db} -} - -func (repository DripFileVowRepository) Create(headerID int64, models []DripFileVowModel) error { - tx, err := repository.db.Begin() +func (repository DripFileVowRepository) Create(headerID int64, models []interface{}) error { + tx, err := repository.DB.Begin() if err != nil { return err } + + var vow DripFileVowModel for _, model := range models { + vow = model.(DripFileVowModel) _, err = tx.Exec( `INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log) - VALUES($1, $2, $3, $4, $5, $6)`, - headerID, model.What, model.Data, model.LogIndex, model.TransactionIndex, model.Raw, + VALUES($1, $2, $3, $4, $5, $6)`, + headerID, vow.What, vow.Data, vow.LogIndex, vow.TransactionIndex, vow.Raw, ) if err != nil { tx.Rollback() return err } } + _, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET drip_file_vow_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_vow_checked = $2`, headerID, true) + if err != nil { tx.Rollback() return err } + return tx.Commit() } func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) + _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) VALUES ($1, $2) - ON CONFLICT (header_id) DO - UPDATE SET drip_file_vow_checked = $2`, headerID, true) + ON CONFLICT (header_id) DO + UPDATE SET drip_file_vow_checked = $2`, headerID, true) return err } func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.db.Select( + err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id + LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR drip_file_vow_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.db.Node.ID, + repository.DB.Node.ID, ) return result, err } + +func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { + repository.DB = db +} diff --git a/pkg/transformers/drip_file/vow/transformer.go b/pkg/transformers/drip_file/vow/transformer.go deleted file mode 100644 index bca52247..00000000 --- a/pkg/transformers/drip_file/vow/transformer.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package vow - -import ( - "log" - - "github.com/ethereum/go-ethereum/common" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type DripFileVowTransformerInitializer struct { - Config shared.TransformerConfig -} - -func (initializer DripFileVowTransformerInitializer) NewDripFileVowTransformer(db *postgres.DB, blockChain core.BlockChain) shared.Transformer { - converter := DripFileVowConverter{} - fetcher := shared.NewFetcher(blockChain) - repository := NewDripFileVowRepository(db) - return DripFileVowTransformer{ - Config: initializer.Config, - Converter: converter, - Fetcher: fetcher, - Repository: repository, - } -} - -type DripFileVowTransformer struct { - Config shared.TransformerConfig - Converter Converter - Fetcher shared.LogFetcher - Repository Repository -} - -func (transformer DripFileVowTransformer) Execute() error { - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - return err - } - log.Printf("Fetching drip file vow event logs for %d headers \n", len(missingHeaders)) - for _, header := range missingHeaders { - topics := [][]common.Hash{{common.HexToHash(shared.DripFileVowSignature)}} - matchingLogs, err := transformer.Fetcher.FetchLogs(drip_file.DripFileConfig.ContractAddresses, topics, header.BlockNumber) - if err != nil { - return err - } - if len(matchingLogs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - return err - } - } - model, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - return err - } - err = transformer.Repository.Create(header.Id, model) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 1612535e..f88b5e29 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -23,7 +23,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/deal" "github.com/vulcanize/vulcanizedb/pkg/transformers/dent" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_drip" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" ilk2 "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" @@ -49,31 +48,46 @@ import ( ) var ( - BiteTransformerInitializer = bite.BiteTransformerInitializer{Config: bite.BiteConfig}.NewBiteTransformer - catFileConfig = cat_file.CatFileConfig - CatFileChopLumpTransformerInitializer = chop_lump.CatFileChopLumpTransformerInitializer{Config: catFileConfig}.NewCatFileChopLumpTransformer - CatFileFlipTransformerInitializer = flip.CatFileFlipTransformerInitializer{Config: catFileConfig}.NewCatFileFlipTransformer - CatFilePitVowTransformerInitializer = pit_vow.CatFilePitVowTransformerInitializer{Config: catFileConfig}.NewCatFilePitVowTransformer - DealTransformerInitializer = deal.DealTransformerInitializer{Config: deal.Config}.NewDealTransformer - DentTransformerInitializer = dent.DentTransformerInitializer{Config: dent.DentConfig}.NewDentTransformer - DripDripTransformerInitializer = drip_drip.DripDripTransformerInitializer{Config: drip_drip.DripDripConfig}.NewDripDripTransformer - dripFileConfig = drip_file.DripFileConfig - DripFileIlkTransformerInitializer = ilk2.DripFileIlkTransformerInitializer{Config: dripFileConfig}.NewDripFileIlkTransformer - DripFileRepoTransformerInitializer = repo.DripFileRepoTransformerInitializer{Config: dripFileConfig}.NewDripFileRepoTransformer - DripFileVowTransfromerInitializer = vow.DripFileVowTransformerInitializer{Config: dripFileConfig}.NewDripFileVowTransformer - FlipKickTransformerInitializer = flip_kick.FlipKickTransformerInitializer{Config: flip_kick.FlipKickConfig}.NewFlipKickTransformer - FlopKickTransformerInitializer = flop_kick.FlopKickTransformerInitializer{Config: flop_kick.Config}.NewFlopKickTransformer - FrobTransformerInitializer = frob.FrobTransformerInitializer{Config: frob.FrobConfig}.NewFrobTransformer - PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer - VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer - PitFileDebtCeilingTransformerInitializer = factories.Transformer{ + BiteTransformerInitializer = bite.BiteTransformerInitializer{Config: bite.BiteConfig}.NewBiteTransformer + catFileConfig = cat_file.CatFileConfig + CatFileChopLumpTransformerInitializer = chop_lump.CatFileChopLumpTransformerInitializer{Config: catFileConfig}.NewCatFileChopLumpTransformer + CatFileFlipTransformerInitializer = flip.CatFileFlipTransformerInitializer{Config: catFileConfig}.NewCatFileFlipTransformer + CatFilePitVowTransformerInitializer = pit_vow.CatFilePitVowTransformerInitializer{Config: catFileConfig}.NewCatFilePitVowTransformer + DealTransformerInitializer = deal.DealTransformerInitializer{Config: deal.Config}.NewDealTransformer + DentTransformerInitializer = dent.DentTransformerInitializer{Config: dent.DentConfig}.NewDentTransformer + DripDripTransformerInitializer = drip_drip.DripDripTransformerInitializer{Config: drip_drip.DripDripConfig}.NewDripDripTransformer + DripFileIlkTransformerInitializer = factories.Transformer{ + Config: ilk2.DripFileIlkConfig, + Converter: &ilk2.DripFileIlkConverter{}, + Repository: &ilk2.DripFileIlkRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + + DripFileRepoTransformerInitializer = factories.Transformer{ + Config: repo.DripFileRepoConfig, + Converter: &repo.DripFileRepoConverter{}, + Repository: &repo.DripFileRepoRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + + DripFileVowTransfromerInitializer = factories.Transformer{ + Config: vow.DripFileVowConfig, + Converter: &vow.DripFileVowConverter{}, + Repository: &vow.DripFileVowRepository{}, + Fetcher: &shared.Fetcher{}, + }.NewTransformer + + FlipKickTransformerInitializer = flip_kick.FlipKickTransformerInitializer{Config: flip_kick.FlipKickConfig}.NewFlipKickTransformer + FlopKickTransformerInitializer = flop_kick.FlopKickTransformerInitializer{Config: flop_kick.Config}.NewFlopKickTransformer + FrobTransformerInitializer = frob.FrobTransformerInitializer{Config: frob.FrobConfig}.NewFrobTransformer + PitFileDebtCeilingTransformerInitializer = factories.Transformer{ Config: debt_ceiling.DebtCeilingFileConfig, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, Fetcher: &shared.Fetcher{}, }.NewTransformer - PitFileIlkTransformerInitializer = factories.Transformer{ + PitFileIlkTransformerInitializer = factories.Transformer{ Config: ilk.IlkFileConfig, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, @@ -87,6 +101,7 @@ var ( Fetcher: &shared.Fetcher{}, }.NewTransformer + PriceFeedTransformerInitializer = price_feeds.PriceFeedTransformerInitializer{Config: price_feeds.PriceFeedConfig}.NewPriceFeedTransformer TendTransformerInitializer = factories.Transformer{ Config: tend.TendConfig, Converter: &tend.TendConverter{}, @@ -100,15 +115,15 @@ var ( Repository: &vat_init.VatInitRepository{}, Fetcher: &shared.Fetcher{}, }.NewTransformer - + VatGrabTransformerInitializer = vat_grab.VatGrabTransformerInitializer{Config: vat_grab.VatGrabConfig}.NewVatGrabTransformer + VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer + VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer VatMoveTransformerInitializer = factories.Transformer{ Config: vat_move.VatMoveConfig, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, Fetcher: &shared.Fetcher{}, }.NewTransformer - VatHealTransformerInitializer = vat_heal.VatHealTransformerInitializer{Config: vat_heal.VatHealConfig}.NewVatHealTransformer - VatFoldTransformerInitializer = vat_fold.VatFoldTransformerInitializer{Config: vat_fold.VatFoldConfig}.NewVatFoldTransformer VatSlipTransformerInitializer = vat_slip.VatSlipTransformerInitializer{Config: vat_slip.VatSlipConfig}.NewVatSlipTransformer VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer From 2a9fafca08cb35065b0f9ed32d4cf85c234ac098 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 18 Oct 2018 11:53:34 +0200 Subject: [PATCH 24/36] Update drip_file tests --- .../drip_file/ilk/converter_test.go | 2 +- .../drip_file/ilk/repository_test.go | 25 ++-- .../drip_file/ilk/transformer_test.go | 141 ++++++------------ .../drip_file/repo/converter_test.go | 2 +- .../drip_file/repo/repository_test.go | 20 ++- .../drip_file/repo/transformer_test.go | 141 ++++++------------ .../drip_file/vow/converter_test.go | 2 +- .../drip_file/vow/repository_test.go | 20 ++- .../drip_file/vow/transformer_test.go | 141 ++++++------------ .../mocks/drip_file/ilk/converter.go | 11 +- .../mocks/drip_file/ilk/repository.go | 30 ++-- .../mocks/drip_file/repo/converter.go | 11 +- .../mocks/drip_file/repo/repository.go | 31 ++-- .../mocks/drip_file/vow/converter.go | 5 +- .../mocks/drip_file/vow/repository.go | 30 ++-- .../mocks/pit_file/ilk/repository.go | 2 +- 16 files changed, 237 insertions(+), 377 deletions(-) diff --git a/pkg/transformers/drip_file/ilk/converter_test.go b/pkg/transformers/drip_file/ilk/converter_test.go index f7f48c9c..7d60ddc5 100644 --- a/pkg/transformers/drip_file/ilk/converter_test.go +++ b/pkg/transformers/drip_file/ilk/converter_test.go @@ -56,6 +56,6 @@ var _ = Describe("Drip file ilk converter", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(models)).To(Equal(1)) - Expect(models[0]).To(Equal(test_data.DripFileIlkModel)) + Expect(models[0].(ilk.DripFileIlkModel)).To(Equal(test_data.DripFileIlkModel)) }) }) diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index 359fc518..6ea5b9e1 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -16,10 +16,8 @@ package ilk_test import ( "database/sql" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -33,7 +31,7 @@ import ( var _ = Describe("Drip file ilk repository", func() { var ( db *postgres.DB - dripFileIlkRepository ilk.Repository + dripFileIlkRepository ilk.DripFileIlkRepository err error headerRepository datastore.HeaderRepository ) @@ -42,7 +40,7 @@ var _ = Describe("Drip file ilk repository", func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - dripFileIlkRepository = ilk.NewDripFileIlkRepository(db) + dripFileIlkRepository = ilk.DripFileIlkRepository{DB: db} }) Describe("Create", func() { @@ -52,7 +50,7 @@ var _ = Describe("Drip file ilk repository", func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) + err = dripFileIlkRepository.Create(headerID, []interface{}{test_data.DripFileIlkModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -69,7 +67,6 @@ var _ = Describe("Drip file ilk repository", func() { }) It("marks header as checked for logs", func() { - Expect(err).NotTo(HaveOccurred()) var headerChecked bool err = db.Get(&headerChecked, `SELECT drip_file_ilk_checked FROM public.checked_headers WHERE header_id = $1`, headerID) Expect(err).NotTo(HaveOccurred()) @@ -77,7 +74,7 @@ var _ = Describe("Drip file ilk repository", func() { }) It("does not duplicate drip file events", func() { - err = dripFileIlkRepository.Create(headerID, []ilk.DripFileIlkModel{test_data.DripFileIlkModel}) + err = dripFileIlkRepository.Create(headerID, []interface{}{test_data.DripFileIlkModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -139,12 +136,12 @@ var _ = Describe("Drip file ilk repository", func() { blockNumbers = []int64{startingBlock, dripFileBlock, endingBlock, endingBlock + 1} headerIDs = []int64{} + for _, n := range blockNumbers { headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) headerIDs = append(headerIDs, headerID) } - }) It("returns headers with no associated drip file event", func() { @@ -179,7 +176,7 @@ var _ = Describe("Drip file ilk repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - dripFileIlkRepositoryTwo := ilk.NewDripFileIlkRepository(dbTwo) + dripFileIlkRepositoryTwo := ilk.DripFileIlkRepository{DB: dbTwo} err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -192,4 +189,14 @@ var _ = Describe("Drip file ilk repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) + + Describe("SetDB", func() { + It("sets the repository db", func() { + db := test_config.NewTestDB(core.Node{}) + repository := ilk.DripFileIlkRepository{} + Expect(repository.DB).To(BeNil()) + repository.SetDB(db) + Expect(repository.DB).To(Equal(db)) + }) + }) }) diff --git a/pkg/transformers/drip_file/ilk/transformer_test.go b/pkg/transformers/drip_file/ilk/transformer_test.go index 3ad7194a..8089fb67 100644 --- a/pkg/transformers/drip_file/ilk/transformer_test.go +++ b/pkg/transformers/drip_file/ilk/transformer_test.go @@ -19,10 +19,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -31,31 +31,40 @@ import ( ) var _ = Describe("Drip file ilk transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &ilk_mocks.MockDripFileIlkRepository{} - transformer := ilk.DripFileIlkTransformer{ - Config: drip_file.DripFileConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &ilk_mocks.MockDripFileIlkConverter{}, - Repository: repository, - } + var ( + config = ilk.DripFileIlkConfig + fetcher mocks.MockLogFetcher + converter ilk_mocks.MockDripFileIlkConverter + repository ilk_mocks.MockDripFileIlkRepository + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) + BeforeEach(func() { + fetcher = mocks.MockLogFetcher{} + converter = ilk_mocks.MockDripFileIlkConverter{} + repository = ilk_mocks.MockDripFileIlkRepository{} + transformer = factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, + } + headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + }) + + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(drip_file.DripFileConfig.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(drip_file.DripFileConfig.EndingBlockNumber)) + Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) }) It("returns error if repository returns error for missing headers", func() { - repository := &ilk_mocks.MockDripFileIlkRepository{} - repository.SetMissingHeadersErr(fakes.FakeError) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &ilk_mocks.MockDripFileIlkConverter{}, - Repository: repository, - } - + repository.SetMissingHeadersError(fakes.FakeError) err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -63,33 +72,18 @@ var _ = Describe("Drip file ilk transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &ilk_mocks.MockDripFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: fetcher, - Converter: &ilk_mocks.MockDripFileIlkConverter{}, - Repository: repository, - } - + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{drip_file.DripFileConfig.ContractAddresses, drip_file.DripFileConfig.ContractAddresses})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.DripFileIlkSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &ilk_mocks.MockDripFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: fetcher, - Converter: &ilk_mocks.MockDripFileIlkConverter{}, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -98,34 +92,16 @@ var _ = Describe("Drip file ilk transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &ilk_mocks.MockDripFileIlkConverter{} - mockRepository := &ilk_mocks.MockDripFileIlkRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := ilk.DripFileIlkTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } - + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &ilk_mocks.MockDripFileIlkConverter{} - mockRepository := &ilk_mocks.MockDripFileIlkRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := ilk.DripFileIlkTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedError(fakes.FakeError) err := transformer.Execute() @@ -134,16 +110,8 @@ var _ = Describe("Drip file ilk transformer", func() { }) It("converts matching logs", func() { - converter := &ilk_mocks.MockDripFileIlkConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileIlkLog}) - repository := &ilk_mocks.MockDripFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -152,17 +120,9 @@ var _ = Describe("Drip file ilk transformer", func() { }) It("returns error if converter returns error", func() { - converter := &ilk_mocks.MockDripFileIlkConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileIlkLog}) - repository := &ilk_mocks.MockDripFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -171,37 +131,20 @@ var _ = Describe("Drip file ilk transformer", func() { }) It("persists drip file model", func() { - converter := &ilk_mocks.MockDripFileIlkConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileIlkLog}) - repository := &ilk_mocks.MockDripFileIlkRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]ilk.DripFileIlkModel{test_data.DripFileIlkModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.DripFileIlkModel})) }) It("returns error if repository returns error for create", func() { - converter := &ilk_mocks.MockDripFileIlkConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileIlkLog}) - repository := &ilk_mocks.MockDripFileIlkRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := ilk.DripFileIlkTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() diff --git a/pkg/transformers/drip_file/repo/converter_test.go b/pkg/transformers/drip_file/repo/converter_test.go index 21ebc9fc..f2c9d223 100644 --- a/pkg/transformers/drip_file/repo/converter_test.go +++ b/pkg/transformers/drip_file/repo/converter_test.go @@ -44,6 +44,6 @@ var _ = Describe("Drip file repo converter", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(models)).To(Equal(1)) - Expect(models[0]).To(Equal(test_data.DripFileRepoModel)) + Expect(models[0].(repo.DripFileRepoModel)).To(Equal(test_data.DripFileRepoModel)) }) }) diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 9973c0ac..91237f94 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -33,7 +33,7 @@ import ( var _ = Describe("Drip file repo repository", func() { var ( db *postgres.DB - dripFileRepoRepository repo.Repository + dripFileRepoRepository repo.DripFileRepoRepository err error headerRepository datastore.HeaderRepository ) @@ -42,7 +42,7 @@ var _ = Describe("Drip file repo repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - dripFileRepoRepository = repo.NewDripFileRepoRepository(db) + dripFileRepoRepository = repo.DripFileRepoRepository{DB: db} }) Describe("Create", func() { @@ -52,7 +52,7 @@ var _ = Describe("Drip file repo repository", func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) + err = dripFileRepoRepository.Create(headerID, []interface{}{test_data.DripFileRepoModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -75,7 +75,7 @@ var _ = Describe("Drip file repo repository", func() { }) It("does not duplicate drip file events", func() { - err = dripFileRepoRepository.Create(headerID, []repo.DripFileRepoModel{test_data.DripFileRepoModel}) + err = dripFileRepoRepository.Create(headerID, []interface{}{test_data.DripFileRepoModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -176,7 +176,7 @@ var _ = Describe("Drip file repo repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - dripFileRepoRepositoryTwo := repo.NewDripFileRepoRepository(dbTwo) + dripFileRepoRepositoryTwo := repo.DripFileRepoRepository{DB: dbTwo} err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -189,4 +189,14 @@ var _ = Describe("Drip file repo repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) + + Describe("SetDB", func() { + It("sets the repository db", func() { + db := test_config.NewTestDB(core.Node{}) + repository := repo.DripFileRepoRepository{} + Expect(repository.DB).To(BeNil()) + repository.SetDB(db) + Expect(repository.DB).To(Equal(db)) + }) + }) }) diff --git a/pkg/transformers/drip_file/repo/transformer_test.go b/pkg/transformers/drip_file/repo/transformer_test.go index 41fe09b8..a98a06bf 100644 --- a/pkg/transformers/drip_file/repo/transformer_test.go +++ b/pkg/transformers/drip_file/repo/transformer_test.go @@ -19,10 +19,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -31,31 +31,40 @@ import ( ) var _ = Describe("Drip file repo transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &repo_mocks.MockDripFileRepoRepository{} - transformer := repo.DripFileRepoTransformer{ - Config: drip_file.DripFileConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &repo_mocks.MockDripFileRepoConverter{}, - Repository: repository, - } + var ( + config = repo.DripFileRepoConfig + fetcher mocks.MockLogFetcher + converter repo_mocks.MockDripFileRepoConverter + repository repo_mocks.MockDripFileRepoRepository + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) + BeforeEach(func() { + fetcher = mocks.MockLogFetcher{} + converter = repo_mocks.MockDripFileRepoConverter{} + repository = repo_mocks.MockDripFileRepoRepository{} + transformer = factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, + } + headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + }) + + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(drip_file.DripFileConfig.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(drip_file.DripFileConfig.EndingBlockNumber)) + Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) }) It("returns error if repository returns error for missing headers", func() { - repository := &repo_mocks.MockDripFileRepoRepository{} - repository.SetMissingHeadersErr(fakes.FakeError) - transformer := repo.DripFileRepoTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &repo_mocks.MockDripFileRepoConverter{}, - Repository: repository, - } - + repository.SetMissingHeadersError(fakes.FakeError) err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -63,33 +72,18 @@ var _ = Describe("Drip file repo transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &repo_mocks.MockDripFileRepoRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := repo.DripFileRepoTransformer{ - Fetcher: fetcher, - Converter: &repo_mocks.MockDripFileRepoConverter{}, - Repository: repository, - } - + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{drip_file.DripFileConfig.ContractAddresses, drip_file.DripFileConfig.ContractAddresses})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.DripFileRepoSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &repo_mocks.MockDripFileRepoRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := repo.DripFileRepoTransformer{ - Fetcher: fetcher, - Converter: &repo_mocks.MockDripFileRepoConverter{}, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -98,34 +92,16 @@ var _ = Describe("Drip file repo transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &repo_mocks.MockDripFileRepoConverter{} - mockRepository := &repo_mocks.MockDripFileRepoRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := repo.DripFileRepoTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } - + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &repo_mocks.MockDripFileRepoConverter{} - mockRepository := &repo_mocks.MockDripFileRepoRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := repo.DripFileRepoTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedError(fakes.FakeError) err := transformer.Execute() @@ -134,16 +110,8 @@ var _ = Describe("Drip file repo transformer", func() { }) It("converts matching logs", func() { - converter := &repo_mocks.MockDripFileRepoConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileRepoLog}) - repository := &repo_mocks.MockDripFileRepoRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := repo.DripFileRepoTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -152,17 +120,9 @@ var _ = Describe("Drip file repo transformer", func() { }) It("returns error if converter returns error", func() { - converter := &repo_mocks.MockDripFileRepoConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileRepoLog}) - repository := &repo_mocks.MockDripFileRepoRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := repo.DripFileRepoTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -171,37 +131,20 @@ var _ = Describe("Drip file repo transformer", func() { }) It("persists drip file model", func() { - converter := &repo_mocks.MockDripFileRepoConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileRepoLog}) - repository := &repo_mocks.MockDripFileRepoRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := repo.DripFileRepoTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]repo.DripFileRepoModel{test_data.DripFileRepoModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.DripFileRepoModel})) }) It("returns error if repository returns error for create", func() { - converter := &repo_mocks.MockDripFileRepoConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileRepoLog}) - repository := &repo_mocks.MockDripFileRepoRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := repo.DripFileRepoTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() diff --git a/pkg/transformers/drip_file/vow/converter_test.go b/pkg/transformers/drip_file/vow/converter_test.go index 0458c611..8678b597 100644 --- a/pkg/transformers/drip_file/vow/converter_test.go +++ b/pkg/transformers/drip_file/vow/converter_test.go @@ -44,6 +44,6 @@ var _ = Describe("Drip file repo converter", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(models)).To(Equal(1)) - Expect(models[0]).To(Equal(test_data.DripFileVowModel)) + Expect(models[0].(vow.DripFileVowModel)).To(Equal(test_data.DripFileVowModel)) }) }) diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index 96fde281..42200cf8 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -33,7 +33,7 @@ import ( var _ = Describe("Drip file vow repository", func() { var ( db *postgres.DB - dripFileVowRepository vow.Repository + dripFileVowRepository vow.DripFileVowRepository err error headerRepository datastore.HeaderRepository ) @@ -42,7 +42,7 @@ var _ = Describe("Drip file vow repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - dripFileVowRepository = vow.NewDripFileVowRepository(db) + dripFileVowRepository = vow.DripFileVowRepository{DB: db} }) Describe("Create", func() { @@ -52,7 +52,7 @@ var _ = Describe("Drip file vow repository", func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) + err = dripFileVowRepository.Create(headerID, []interface{}{test_data.DripFileVowModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -75,7 +75,7 @@ var _ = Describe("Drip file vow repository", func() { }) It("does not duplicate drip file events", func() { - err = dripFileVowRepository.Create(headerID, []vow.DripFileVowModel{test_data.DripFileVowModel}) + err = dripFileVowRepository.Create(headerID, []interface{}{test_data.DripFileVowModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) @@ -176,7 +176,7 @@ var _ = Describe("Drip file vow repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - dripFileVowRepositoryTwo := vow.NewDripFileVowRepository(dbTwo) + dripFileVowRepositoryTwo := vow.DripFileVowRepository{DB: dbTwo} err := dripFileVowRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -189,4 +189,14 @@ var _ = Describe("Drip file vow repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) + + Describe("SetDB", func() { + It("sets the repository db", func() { + db := test_config.NewTestDB(core.Node{}) + repository := vow.DripFileVowRepository{} + Expect(repository.DB).To(BeNil()) + repository.SetDB(db) + Expect(repository.DB).To(Equal(db)) + }) + }) }) diff --git a/pkg/transformers/drip_file/vow/transformer_test.go b/pkg/transformers/drip_file/vow/transformer_test.go index eb55c3e1..4d9e7d23 100644 --- a/pkg/transformers/drip_file/vow/transformer_test.go +++ b/pkg/transformers/drip_file/vow/transformer_test.go @@ -19,10 +19,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" @@ -31,31 +31,40 @@ import ( ) var _ = Describe("Drip file vow transformer", func() { - It("gets missing headers for block numbers specified in config", func() { - repository := &vow_mocks.MockDripFileVowRepository{} - transformer := vow.DripFileVowTransformer{ - Config: drip_file.DripFileConfig, - Fetcher: &mocks.MockLogFetcher{}, - Converter: &vow_mocks.MockDripFileVowConverter{}, - Repository: repository, - } + var ( + config = vow.DripFileVowConfig + fetcher mocks.MockLogFetcher + converter vow_mocks.MockDripFileVowConverter + repository vow_mocks.MockDripFileVowRepository + transformer shared.Transformer + headerOne core.Header + headerTwo core.Header + ) + BeforeEach(func() { + fetcher = mocks.MockLogFetcher{} + converter = vow_mocks.MockDripFileVowConverter{} + repository = vow_mocks.MockDripFileVowRepository{} + transformer = factories.Transformer{ + Config: config, + Fetcher: &fetcher, + Converter: &converter, + Repository: &repository, + } + headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + }) + + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(drip_file.DripFileConfig.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(drip_file.DripFileConfig.EndingBlockNumber)) + Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) + Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) }) It("returns error if repository returns error for missing headers", func() { - repository := &vow_mocks.MockDripFileVowRepository{} - repository.SetMissingHeadersErr(fakes.FakeError) - transformer := vow.DripFileVowTransformer{ - Fetcher: &mocks.MockLogFetcher{}, - Converter: &vow_mocks.MockDripFileVowConverter{}, - Repository: repository, - } - + repository.SetMissingHeadersError(fakes.FakeError) err := transformer.Execute() Expect(err).To(HaveOccurred()) @@ -63,33 +72,18 @@ var _ = Describe("Drip file vow transformer", func() { }) It("fetches logs for missing headers", func() { - fetcher := &mocks.MockLogFetcher{} - repository := &vow_mocks.MockDripFileVowRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}, {BlockNumber: 2}}) - transformer := vow.DripFileVowTransformer{ - Fetcher: fetcher, - Converter: &vow_mocks.MockDripFileVowConverter{}, - Repository: repository, - } - + repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{1, 2})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{drip_file.DripFileConfig.ContractAddresses, drip_file.DripFileConfig.ContractAddresses})) + Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) + Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{common.HexToHash(shared.DripFileVowSignature)}})) }) It("returns error if fetcher returns error", func() { - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetcherError(fakes.FakeError) - repository := &vow_mocks.MockDripFileVowRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vow.DripFileVowTransformer{ - Fetcher: fetcher, - Converter: &vow_mocks.MockDripFileVowConverter{}, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -98,34 +92,16 @@ var _ = Describe("Drip file vow transformer", func() { }) It("marks header checked if no logs returned", func() { - mockConverter := &vow_mocks.MockDripFileVowConverter{} - mockRepository := &vow_mocks.MockDripFileVowRepository{} - headerID := int64(123) - mockRepository.SetMissingHeaders([]core.Header{{Id: headerID}}) - mockFetcher := &mocks.MockLogFetcher{} - transformer := vow.DripFileVowTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } - + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - mockRepository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("returns error if marking header checked returns err", func() { - mockConverter := &vow_mocks.MockDripFileVowConverter{} - mockRepository := &vow_mocks.MockDripFileVowRepository{} - mockRepository.SetMissingHeaders([]core.Header{{Id: int64(123)}}) - mockRepository.SetMarkHeaderCheckedErr(fakes.FakeError) - mockFetcher := &mocks.MockLogFetcher{} - transformer := vow.DripFileVowTransformer{ - Converter: mockConverter, - Fetcher: mockFetcher, - Repository: mockRepository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) + repository.SetMarkHeaderCheckedError(fakes.FakeError) err := transformer.Execute() @@ -134,16 +110,8 @@ var _ = Describe("Drip file vow transformer", func() { }) It("converts matching logs", func() { - converter := &vow_mocks.MockDripFileVowConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileVowLog}) - repository := &vow_mocks.MockDripFileVowRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vow.DripFileVowTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -152,17 +120,9 @@ var _ = Describe("Drip file vow transformer", func() { }) It("returns error if converter returns error", func() { - converter := &vow_mocks.MockDripFileVowConverter{} converter.SetConverterError(fakes.FakeError) - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileVowLog}) - repository := &vow_mocks.MockDripFileVowRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1}}) - transformer := vow.DripFileVowTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() @@ -171,37 +131,20 @@ var _ = Describe("Drip file vow transformer", func() { }) It("persists drip file model", func() { - converter := &vow_mocks.MockDripFileVowConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileVowLog}) - repository := &vow_mocks.MockDripFileVowRepository{} - fakeHeader := core.Header{BlockNumber: 1, Id: 2} - repository.SetMissingHeaders([]core.Header{fakeHeader}) - transformer := vow.DripFileVowTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } + repository.SetMissingHeaders([]core.Header{headerOne}) err := transformer.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedHeaderID).To(Equal(fakeHeader.Id)) - Expect(repository.PassedModels).To(Equal([]vow.DripFileVowModel{test_data.DripFileVowModel})) + Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) + Expect(repository.PassedModels).To(Equal([]interface{}{test_data.DripFileVowModel})) }) It("returns error if repository returns error for create", func() { - converter := &vow_mocks.MockDripFileVowConverter{} - fetcher := &mocks.MockLogFetcher{} fetcher.SetFetchedLogs([]types.Log{test_data.EthDripFileVowLog}) - repository := &vow_mocks.MockDripFileVowRepository{} - repository.SetMissingHeaders([]core.Header{{BlockNumber: 1, Id: 2}}) + repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - transformer := vow.DripFileVowTransformer{ - Fetcher: fetcher, - Converter: converter, - Repository: repository, - } err := transformer.Execute() diff --git a/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go b/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go index 85492371..0ab9ace0 100644 --- a/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go +++ b/pkg/transformers/test_data/mocks/drip_file/ilk/converter.go @@ -16,20 +16,19 @@ package ilk import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) type MockDripFileIlkConverter struct { - converterErr error - PassedLogs []types.Log + converterError error + PassedLogs []types.Log } -func (converter *MockDripFileIlkConverter) ToModels(ethLogs []types.Log) ([]ilk.DripFileIlkModel, error) { +func (converter *MockDripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []ilk.DripFileIlkModel{test_data.DripFileIlkModel}, converter.converterErr + return []interface{}{test_data.DripFileIlkModel}, converter.converterError } func (converter *MockDripFileIlkConverter) SetConverterError(e error) { - converter.converterErr = e + converter.converterError = e } diff --git a/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go b/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go index b76aa85e..fe96b361 100644 --- a/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go @@ -16,46 +16,46 @@ package ilk import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/ilk" ) type MockDripFileIlkRepository struct { - createErr error - markHeaderCheckedErr error + createError error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []ilk.DripFileIlkModel + PassedModels []interface{} } -func (repository *MockDripFileIlkRepository) Create(headerID int64, models []ilk.DripFileIlkModel) error { +func (repository *MockDripFileIlkRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models - return repository.createErr + return repository.createError } func (repository *MockDripFileIlkRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockDripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } -func (repository *MockDripFileIlkRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e +func (repository *MockDripFileIlkRepository) SetMarkHeaderCheckedError(e error) { + repository.markHeaderCheckedError = e } -func (repository *MockDripFileIlkRepository) SetMissingHeadersErr(e error) { - repository.missingHeadersErr = e +func (repository *MockDripFileIlkRepository) SetMissingHeadersError(e error) { + repository.missingHeadersError = e } func (repository *MockDripFileIlkRepository) SetMissingHeaders(headers []core.Header) { @@ -63,8 +63,10 @@ func (repository *MockDripFileIlkRepository) SetMissingHeaders(headers []core.He } func (repository *MockDripFileIlkRepository) SetCreateError(e error) { - repository.createErr = e + repository.createError = e } func (repository *MockDripFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockDripFileIlkRepository) SetDB(db *postgres.DB) {} diff --git a/pkg/transformers/test_data/mocks/drip_file/repo/converter.go b/pkg/transformers/test_data/mocks/drip_file/repo/converter.go index 0dbe80d0..528fef56 100644 --- a/pkg/transformers/test_data/mocks/drip_file/repo/converter.go +++ b/pkg/transformers/test_data/mocks/drip_file/repo/converter.go @@ -16,20 +16,19 @@ package repo import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) type MockDripFileRepoConverter struct { - converterErr error - PassedLogs []types.Log + converterError error + PassedLogs []types.Log } -func (converter *MockDripFileRepoConverter) ToModels(ethLogs []types.Log) ([]repo.DripFileRepoModel, error) { +func (converter *MockDripFileRepoConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []repo.DripFileRepoModel{test_data.DripFileRepoModel}, converter.converterErr + return []interface{}{test_data.DripFileRepoModel}, converter.converterError } func (converter *MockDripFileRepoConverter) SetConverterError(e error) { - converter.converterErr = e + converter.converterError = e } diff --git a/pkg/transformers/test_data/mocks/drip_file/repo/repository.go b/pkg/transformers/test_data/mocks/drip_file/repo/repository.go index be03693a..40131fff 100644 --- a/pkg/transformers/test_data/mocks/drip_file/repo/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/repo/repository.go @@ -16,46 +16,46 @@ package repo import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/repo" ) type MockDripFileRepoRepository struct { - createErr error - markHeaderCheckedErr error + createError error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []repo.DripFileRepoModel + PassedModels []interface{} } -func (repository *MockDripFileRepoRepository) Create(headerID int64, models []repo.DripFileRepoModel) error { +func (repository *MockDripFileRepoRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models - return repository.createErr + return repository.createError } func (repository *MockDripFileRepoRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockDripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } -func (repository *MockDripFileRepoRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e +func (repository *MockDripFileRepoRepository) SetMarkHeaderCheckedError(e error) { + repository.markHeaderCheckedError = e } -func (repository *MockDripFileRepoRepository) SetMissingHeadersErr(e error) { - repository.missingHeadersErr = e +func (repository *MockDripFileRepoRepository) SetMissingHeadersError(e error) { + repository.missingHeadersError = e } func (repository *MockDripFileRepoRepository) SetMissingHeaders(headers []core.Header) { @@ -63,8 +63,11 @@ func (repository *MockDripFileRepoRepository) SetMissingHeaders(headers []core.H } func (repository *MockDripFileRepoRepository) SetCreateError(e error) { - repository.createErr = e + repository.createError = e } + func (repository *MockDripFileRepoRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockDripFileRepoRepository) SetDB(db *postgres.DB) {} diff --git a/pkg/transformers/test_data/mocks/drip_file/vow/converter.go b/pkg/transformers/test_data/mocks/drip_file/vow/converter.go index 7d7adb29..4c9c0248 100644 --- a/pkg/transformers/test_data/mocks/drip_file/vow/converter.go +++ b/pkg/transformers/test_data/mocks/drip_file/vow/converter.go @@ -16,7 +16,6 @@ package vow import ( "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" ) @@ -25,9 +24,9 @@ type MockDripFileVowConverter struct { PassedLogs []types.Log } -func (converter *MockDripFileVowConverter) ToModels(ethLogs []types.Log) ([]vow.DripFileVowModel, error) { +func (converter *MockDripFileVowConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { converter.PassedLogs = ethLogs - return []vow.DripFileVowModel{test_data.DripFileVowModel}, converter.converterErr + return []interface{}{test_data.DripFileVowModel}, converter.converterErr } func (converter *MockDripFileVowConverter) SetConverterError(e error) { diff --git a/pkg/transformers/test_data/mocks/drip_file/vow/repository.go b/pkg/transformers/test_data/mocks/drip_file/vow/repository.go index 54e36edd..e365817a 100644 --- a/pkg/transformers/test_data/mocks/drip_file/vow/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/vow/repository.go @@ -16,46 +16,46 @@ package vow import ( . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" ) type MockDripFileVowRepository struct { - createErr error - markHeaderCheckedErr error + createError error + markHeaderCheckedError error markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header - missingHeadersErr error + missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 - PassedModels []vow.DripFileVowModel + PassedModels []interface{} } -func (repository *MockDripFileVowRepository) Create(headerID int64, models []vow.DripFileVowModel) error { +func (repository *MockDripFileVowRepository) Create(headerID int64, models []interface{}) error { repository.PassedHeaderID = headerID repository.PassedModels = models - return repository.createErr + return repository.createError } func (repository *MockDripFileVowRepository) MarkHeaderChecked(headerID int64) error { repository.markHeaderCheckedPassedHeaderID = headerID - return repository.markHeaderCheckedErr + return repository.markHeaderCheckedError } func (repository *MockDripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { repository.PassedStartingBlockNumber = startingBlockNumber repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersErr + return repository.missingHeaders, repository.missingHeadersError } -func (repository *MockDripFileVowRepository) SetMarkHeaderCheckedErr(e error) { - repository.markHeaderCheckedErr = e +func (repository *MockDripFileVowRepository) SetMarkHeaderCheckedError(e error) { + repository.markHeaderCheckedError = e } -func (repository *MockDripFileVowRepository) SetMissingHeadersErr(e error) { - repository.missingHeadersErr = e +func (repository *MockDripFileVowRepository) SetMissingHeadersError(e error) { + repository.missingHeadersError = e } func (repository *MockDripFileVowRepository) SetMissingHeaders(headers []core.Header) { @@ -63,8 +63,10 @@ func (repository *MockDripFileVowRepository) SetMissingHeaders(headers []core.He } func (repository *MockDripFileVowRepository) SetCreateError(e error) { - repository.createErr = e + repository.createError = e } func (repository *MockDripFileVowRepository) AssertMarkHeaderCheckedCalledWith(i int64) { Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } + +func (repository *MockDripFileVowRepository) SetDB(db *postgres.DB) {} diff --git a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go index 899ffab4..0ab6fc75 100644 --- a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go @@ -70,4 +70,4 @@ func (repository *MockPitFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockPitFileIlkRepository) SetDB(db *postgres.DB) {} \ No newline at end of file +func (repository *MockPitFileIlkRepository) SetDB(db *postgres.DB) {} From 70722fc9e39d5de3e0c639aab46db156e78db821 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 18 Oct 2018 11:55:32 +0200 Subject: [PATCH 25/36] Add todo for changing to single topic in shared transformer config --- pkg/transformers/shared/transformer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index acede32c..6b7909e5 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -31,7 +31,7 @@ type TransformerConfig struct { TransformerName string ContractAddresses []string ContractAbi string - Topics []string + Topics []string // TODO Change this to single topic StartingBlockNumber int64 EndingBlockNumber int64 } From 7d3bf1f0f680c07124ff012a5e354d85882f8699 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 18 Oct 2018 13:15:11 +0200 Subject: [PATCH 26/36] Fixes after rebase --- db/schema.sql | 4 +-- .../pit_file/ilk/repository_test.go | 8 +---- .../pit_file/stability_fee/repository_test.go | 6 +--- pkg/transformers/tend/repository_test.go | 21 +++++------- pkg/transformers/transformers.go | 8 ++--- pkg/transformers/vat_init/repository_test.go | 32 +++---------------- pkg/transformers/vat_move/converter.go | 12 +++---- 7 files changed, 24 insertions(+), 67 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index 1445a09c..ed69d61a 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 10.3 --- Dumped by pg_dump version 10.3 +-- Dumped from database version 10.5 +-- Dumped by pg_dump version 10.5 SET statement_timeout = 0; SET lock_timeout = 0; diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index 1c6a7225..fc40803a 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -46,12 +46,7 @@ var _ = Describe("Pit file ilk repository", func() { }) Describe("Create", func() { - var ( - db *postgres.DB - pitFileIlkRepository ilk.PitFileIlkRepository - err error - headerID int64 - ) + var headerID int64 BeforeEach(func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) @@ -59,7 +54,6 @@ var _ = Describe("Pit file ilk repository", func() { err = pitFileRepository.Create(headerID, []interface{}{test_data.PitFileIlkModel}) Expect(err).NotTo(HaveOccurred()) - pitFileIlkRepository = ilk.PitFileIlkRepository{DB: db} }) It("adds a pit file ilk event", func() { diff --git a/pkg/transformers/pit_file/stability_fee/repository_test.go b/pkg/transformers/pit_file/stability_fee/repository_test.go index 62f3b416..76081079 100644 --- a/pkg/transformers/pit_file/stability_fee/repository_test.go +++ b/pkg/transformers/pit_file/stability_fee/repository_test.go @@ -46,11 +46,7 @@ var _ = Describe("Pit file stability fee repository", func() { }) Describe("Create", func() { - var ( - db *postgres.DB - err error - headerID int64 - ) + var headerID int64 BeforeEach(func() { headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index a6d43f58..86c92e88 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -41,15 +41,15 @@ var _ = Describe("TendRepository", func() { db = test_config.NewTestDB(node) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - headerId, err = headerRepository.CreateOrUpdateHeader(core.Header{}) - Expect(err).NotTo(HaveOccurred()) - tendRepository = tend.TendRepository{DB: db} }) Describe("Create", func() { - It("persists a tend record", func() { - err := tendRepository.Create(headerId, []interface{}{test_data.TendModel}) + BeforeEach(func() { + headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(err).NotTo(HaveOccurred()) + + err = tendRepository.Create(headerId, []interface{}{test_data.TendModel}) Expect(err).NotTo(HaveOccurred()) }) @@ -80,18 +80,12 @@ var _ = Describe("TendRepository", func() { }) It("returns an error if inserting a tend record fails", func() { - err := tendRepository.Create(headerId, []interface{}{test_data.TendModel}) - Expect(err).NotTo(HaveOccurred()) - err = tendRepository.Create(headerId, []interface{}{test_data.TendModel}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) }) It("deletes the tend record if its corresponding header record is deleted", func() { - err := tendRepository.Create(headerId, []interface{}{test_data.TendModel}) - Expect(err).NotTo(HaveOccurred()) - var count int err = db.QueryRow(`SELECT count(*) from maker.tend`).Scan(&count) Expect(err).NotTo(HaveOccurred()) @@ -107,11 +101,12 @@ var _ = Describe("TendRepository", func() { }) Describe("MarkHeaderChecked", func() { - var headerId int64 - BeforeEach(func() { headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) + + err = tendRepository.Create(headerId, []interface{}{test_data.TendModel}) + Expect(err).NotTo(HaveOccurred()) }) It("creates a row for a new headerID", func() { diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index f88b5e29..2cf296ff 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -124,10 +124,10 @@ var ( Repository: &vat_move.VatMoveRepository{}, Fetcher: &shared.Fetcher{}, }.NewTransformer - VatSlipTransformerInitializer = vat_slip.VatSlipTransformerInitializer{Config: vat_slip.VatSlipConfig}.NewVatSlipTransformer - VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer - VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer - VatFluxTransformerInitializer = vat_flux.VatFluxTransformerInitializer{Config: vat_flux.VatFluxConfig}.NewVatFluxTransformer + VatSlipTransformerInitializer = vat_slip.VatSlipTransformerInitializer{Config: vat_slip.VatSlipConfig}.NewVatSlipTransformer + VatTollTransformerInitializer = vat_toll.VatTollTransformerInitializer{Config: vat_toll.VatTollConfig}.NewVatTollTransformer + VatTuneTransformerInitializer = vat_tune.VatTuneTransformerInitializer{Config: vat_tune.VatTuneConfig}.NewVatTuneTransformer + VatFluxTransformerInitializer = vat_flux.VatFluxTransformerInitializer{Config: vat_flux.VatFluxConfig}.NewVatFluxTransformer ) func TransformerInitializers() []shared.TransformerInitializer { diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index 7ce6a647..fad81598 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -65,9 +65,6 @@ var _ = Describe("Vat init repository", func() { }) It("does not duplicate vat events", func() { - err = vatInitRepository.Create(headerID, []interface{}{test_data.VatInitModel}) - Expect(err).NotTo(HaveOccurred()) - err = vatInitRepository.Create(headerID, []interface{}{test_data.VatInitModel}) Expect(err).To(HaveOccurred()) @@ -93,20 +90,11 @@ var _ = Describe("Vat init repository", func() { }) Describe("MarkHeaderChecked", func() { - var ( - db *postgres.DB - vatInitRepository vat_init.VatInitRepository - err error - headerID int64 - ) + var headerID int64 BeforeEach(func() { - db = test_config.NewTestDB(core.Node{}) - test_config.CleanTestDB(db) - headerRepository := repositories.NewHeaderRepository(db) - headerID, err = headerRepository.CreateOrUpdateHeader(core.Header{}) + headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - vatInitRepository = vat_init.VatInitRepository{DB: db} }) It("creates a row for a new headerID", func() { @@ -166,16 +154,6 @@ var _ = Describe("Vat init repository", func() { }) It("only treats headers as checked if drip drip logs have been checked", func() { - startingBlockNumber := int64(1) - vatInitBlockNumber := int64(2) - endingBlockNumber := int64(3) - blockNumbers = []int64{startingBlockNumber, vatInitBlockNumber, endingBlockNumber, endingBlockNumber + 1} - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(core.Header{BlockNumber: n}) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) Expect(err).NotTo(HaveOccurred()) @@ -196,9 +174,8 @@ var _ = Describe("Vat init repository", func() { Expect(err).NotTo(HaveOccurred()) } - vatInitRepository := vat_init.VatInitRepository{DB: db} vatInitRepositoryTwo := vat_init.VatInitRepository{DB: dbTwo} - err := vatInitRepository.MarkHeaderChecked(headerIDs[0]) + err = vatInitRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) nodeOneMissingHeaders, err := vatInitRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) @@ -213,8 +190,7 @@ var _ = Describe("Vat init repository", func() { Describe("SetDB", func() { It("sets the repository db", func() { - db := test_config.NewTestDB(core.Node{}) - vatInitRepository := vat_init.VatInitRepository{DB: nil} + vatInitRepository.SetDB(nil) Expect(vatInitRepository.DB).To(BeNil()) vatInitRepository.SetDB(db) Expect(vatInitRepository.DB).To(Equal(db)) diff --git a/pkg/transformers/vat_move/converter.go b/pkg/transformers/vat_move/converter.go index c28aa71a..4896a786 100644 --- a/pkg/transformers/vat_move/converter.go +++ b/pkg/transformers/vat_move/converter.go @@ -21,18 +21,14 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -type Converter interface { - ToModels(ethLog []types.Log) ([]VatMoveModel, error) -} - type VatMoveConverter struct{} -func (VatMoveConverter) ToModels(ethLogs []types.Log) ([]VatMoveModel, error) { - var models []VatMoveModel +func (VatMoveConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { + var models []interface{} for _, ethLog := range ethLogs { err := verifyLog(ethLog) if err != nil { - return []VatMoveModel{}, err + return []interface{}{}, err } src := common.BytesToAddress(ethLog.Topics[1].Bytes()) @@ -40,7 +36,7 @@ func (VatMoveConverter) ToModels(ethLogs []types.Log) ([]VatMoveModel, error) { rad := ethLog.Topics[3].Big() raw, err := json.Marshal(ethLog) if err != nil { - return []VatMoveModel{}, err + return []interface{}{}, err } models = append(models, VatMoveModel{ From 33f1f62c11dcc2bcc616d99ea8a8ab141039ed2f Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 19 Oct 2018 14:53:02 +0200 Subject: [PATCH 27/36] Implement SingleTransformerConfig --- pkg/transformers/drip_file/ilk/config.go | 4 ++-- pkg/transformers/drip_file/repo/config.go | 4 ++-- pkg/transformers/drip_file/vow/config.go | 4 ++-- pkg/transformers/factories/transformer.go | 14 ++++++-------- pkg/transformers/pit_file/debt_ceiling/config.go | 4 ++-- pkg/transformers/pit_file/ilk/config.go | 4 ++-- pkg/transformers/pit_file/stability_fee/config.go | 4 ++-- pkg/transformers/shared/transformer.go | 9 +++++++++ pkg/transformers/tend/config.go | 4 ++-- pkg/transformers/vat_init/config.go | 4 ++-- pkg/transformers/vat_move/config.go | 4 ++-- 11 files changed, 33 insertions(+), 26 deletions(-) diff --git a/pkg/transformers/drip_file/ilk/config.go b/pkg/transformers/drip_file/ilk/config.go index 9e58fd6e..d4bafbf6 100644 --- a/pkg/transformers/drip_file/ilk/config.go +++ b/pkg/transformers/drip_file/ilk/config.go @@ -16,11 +16,11 @@ package ilk import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var DripFileIlkConfig = shared.TransformerConfig{ +var DripFileIlkConfig = shared.SingleTransformerConfig{ TransformerName: "drip file ilk", ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, - Topics: []string{shared.DripFileIlkSignature}, + Topic: shared.DripFileIlkSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/drip_file/repo/config.go b/pkg/transformers/drip_file/repo/config.go index f5c52fe5..676a8341 100644 --- a/pkg/transformers/drip_file/repo/config.go +++ b/pkg/transformers/drip_file/repo/config.go @@ -16,11 +16,11 @@ package repo import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var DripFileRepoConfig = shared.TransformerConfig{ +var DripFileRepoConfig = shared.SingleTransformerConfig{ TransformerName: "drip file repo", ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, - Topics: []string{shared.DripFileRepoSignature}, + Topic: shared.DripFileRepoSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/drip_file/vow/config.go b/pkg/transformers/drip_file/vow/config.go index 2d7f9004..aeda864f 100644 --- a/pkg/transformers/drip_file/vow/config.go +++ b/pkg/transformers/drip_file/vow/config.go @@ -16,11 +16,11 @@ package vow import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var DripFileVowConfig = shared.TransformerConfig{ +var DripFileVowConfig = shared.SingleTransformerConfig{ TransformerName: "drip file vow", ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, - Topics: []string{shared.DripFileVowSignature}, + Topic: shared.DripFileVowSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 3be68c56..a9304166 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -26,7 +26,7 @@ import ( ) type Transformer struct { - Config shared.TransformerConfig + Config shared.SingleTransformerConfig Converter Converter Repository Repository Fetcher shared.SettableLogFetcher @@ -57,16 +57,14 @@ func (transformer Transformer) Execute() error { return err } + // Grab event signature from transformer config + // (Double-array structure required for go-ethereum FilterQuery) + var topic = [][]common.Hash{{common.HexToHash(transformer.Config.Topic)}} + log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders)) for _, header := range missingHeaders { - // Grab topics from config - var topics [][]common.Hash - for _, topic := range transformer.Config.Topics { - topics = append(topics, []common.Hash{common.HexToHash(topic)}) - } - // Fetch the missing logs for a given header - matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topics, header.BlockNumber) + matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header.BlockNumber) if err != nil { log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) return err diff --git a/pkg/transformers/pit_file/debt_ceiling/config.go b/pkg/transformers/pit_file/debt_ceiling/config.go index 23ee22b4..93c483f3 100644 --- a/pkg/transformers/pit_file/debt_ceiling/config.go +++ b/pkg/transformers/pit_file/debt_ceiling/config.go @@ -18,11 +18,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -var DebtCeilingFileConfig = shared.TransformerConfig{ +var DebtCeilingFileConfig = shared.SingleTransformerConfig{ TransformerName: "pit file debt ceiling", ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, - Topics: []string{shared.PitFileDebtCeilingSignature}, + Topic: shared.PitFileDebtCeilingSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/pit_file/ilk/config.go b/pkg/transformers/pit_file/ilk/config.go index 19d65719..c2def4e2 100644 --- a/pkg/transformers/pit_file/ilk/config.go +++ b/pkg/transformers/pit_file/ilk/config.go @@ -18,11 +18,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -var IlkFileConfig = shared.TransformerConfig{ +var IlkFileConfig = shared.SingleTransformerConfig{ TransformerName: "pit file ilk", ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, - Topics: []string{shared.PitFileIlkSignature}, + Topic: shared.PitFileIlkSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/pit_file/stability_fee/config.go b/pkg/transformers/pit_file/stability_fee/config.go index 1ca34940..d1894d63 100644 --- a/pkg/transformers/pit_file/stability_fee/config.go +++ b/pkg/transformers/pit_file/stability_fee/config.go @@ -18,11 +18,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -var StabilityFeeFileConfig = shared.TransformerConfig{ +var StabilityFeeFileConfig = shared.SingleTransformerConfig{ TransformerName: "pit file stability fee", ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, - Topics: []string{shared.PitFileStabilityFeeSignature}, + Topic: shared.PitFileStabilityFeeSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 6b7909e5..a647e9a7 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -36,6 +36,15 @@ type TransformerConfig struct { EndingBlockNumber int64 } +type SingleTransformerConfig struct { + TransformerName string + ContractAddresses []string + ContractAbi string + Topic string + StartingBlockNumber int64 + EndingBlockNumber int64 +} + func HexToInt64(byteString string) int64 { value := common.HexToHash(byteString) return value.Big().Int64() diff --git a/pkg/transformers/tend/config.go b/pkg/transformers/tend/config.go index 03a15e70..1bd07c86 100644 --- a/pkg/transformers/tend/config.go +++ b/pkg/transformers/tend/config.go @@ -16,11 +16,11 @@ package tend import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -var TendConfig = shared.TransformerConfig{ +var TendConfig = shared.SingleTransformerConfig{ TransformerName: "tend", ContractAddresses: []string{shared.FlipperContractAddress}, ContractAbi: shared.FlipperABI, - Topics: []string{shared.TendFunctionSignature}, + Topic: shared.TendFunctionSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/vat_init/config.go b/pkg/transformers/vat_init/config.go index 2f9abf25..ed5d34d5 100644 --- a/pkg/transformers/vat_init/config.go +++ b/pkg/transformers/vat_init/config.go @@ -18,11 +18,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -var VatInitConfig = shared.TransformerConfig{ +var VatInitConfig = shared.SingleTransformerConfig{ TransformerName: "vat init", ContractAddresses: []string{shared.VatContractAddress}, ContractAbi: shared.VatABI, - Topics: []string{shared.VatInitSignature}, + Topic: shared.VatInitSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } diff --git a/pkg/transformers/vat_move/config.go b/pkg/transformers/vat_move/config.go index a98bc52a..f9da1184 100644 --- a/pkg/transformers/vat_move/config.go +++ b/pkg/transformers/vat_move/config.go @@ -18,11 +18,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -var VatMoveConfig = shared.TransformerConfig{ +var VatMoveConfig = shared.SingleTransformerConfig{ TransformerName: "vat move", ContractAddresses: []string{shared.VatContractAddress}, ContractAbi: shared.VatABI, - Topics: []string{shared.VatMoveSignature}, + Topic: shared.VatMoveSignature, StartingBlockNumber: 0, EndingBlockNumber: 10000000, } From e7e511764e4291b0803d9be4c7cc9f29093ed3dc Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 19 Oct 2018 14:54:26 +0200 Subject: [PATCH 28/36] Implement skip to next header on empty matchingLogs --- pkg/transformers/factories/transformer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index a9304166..8b7574b3 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -77,6 +77,8 @@ func (transformer Transformer) Execute() error { log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err } + // Continue with the next header; nothing to persist + continue } models, err := transformer.Converter.ToModels(matchingLogs) From bb204b6b14a45073c637ed90fc6c74dfbd7dd753 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 19 Oct 2018 15:44:55 +0200 Subject: [PATCH 29/36] Remove unnecessary type washing in factory transformer --- pkg/transformers/factories/transformer.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 8b7574b3..93c7048e 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -87,14 +87,7 @@ func (transformer Transformer) Execute() error { return err } - // Can't assert a whole collection, wash types individually for Create, - // which needs interface{} - var typelessModels []interface{} - for _, m := range models { - typelessModels = append(typelessModels, m.(interface{})) - } - - err = transformer.Repository.Create(header.Id, typelessModels) + err = transformer.Repository.Create(header.Id, models) if err != nil { log.Printf("Error persisting %v record: %v", transformerName, err) return err From b7f8432e65d5358ff156f06e8fd1433be796f151 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 22 Oct 2018 11:06:34 +0200 Subject: [PATCH 30/36] Fix git whitespace warnings --- pkg/transformers/drip_file/ilk/repository.go | 10 +++++----- pkg/transformers/drip_file/repo/repository.go | 10 +++++----- pkg/transformers/drip_file/vow/repository.go | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index dc12720b..78a741d7 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -68,11 +68,11 @@ func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endi err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR drip_file_ilk_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_file_ilk_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, repository.DB.Node.ID, diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 7b28c58f..80a6b21e 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -70,11 +70,11 @@ func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, end err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR drip_file_repo_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_file_repo_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, repository.DB.Node.ID, diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 4aa39940..4d3fdaa3 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -69,11 +69,11 @@ func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endi err := repository.DB.Select( &result, `SELECT headers.id, headers.block_number FROM headers - LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR drip_file_vow_checked IS FALSE) - AND headers.block_number >= $1 - AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3`, + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR drip_file_vow_checked IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, repository.DB.Node.ID, From cb74cbe9907e1fb1917f3d241c9d4cfc7a5072c0 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 22 Oct 2018 14:19:53 +0200 Subject: [PATCH 31/36] Add cast error checking to refactored transformers --- pkg/transformers/drip_file/ilk/repository.go | 9 +++++++-- .../drip_file/ilk/repository_test.go | 6 ++++++ pkg/transformers/drip_file/repo/repository.go | 9 +++++++-- .../drip_file/repo/repository_test.go | 6 ++++++ pkg/transformers/drip_file/vow/repository.go | 9 +++++++-- .../drip_file/vow/repository_test.go | 6 ++++++ .../integration_tests/drip_file_vow.go | 14 +++++++++---- .../pit_file_debt_ceiling.go | 14 +++++++++---- .../integration_tests/pit_file_ilk.go | 14 +++++++++---- .../pit_file_stability_fee.go | 14 +++++++++---- pkg/transformers/integration_tests/tend.go | 20 +++++++++++++++---- .../integration_tests/vat_init.go | 11 ++++++++-- .../integration_tests/vat_move.go | 11 ++++++++-- .../pit_file/debt_ceiling/repository.go | 9 +++++++-- .../pit_file/debt_ceiling/repository_test.go | 6 ++++++ pkg/transformers/pit_file/ilk/repository.go | 9 +++++++-- .../pit_file/ilk/repository_test.go | 6 ++++++ .../pit_file/stability_fee/repository.go | 9 +++++++-- .../pit_file/stability_fee/repository_test.go | 6 ++++++ pkg/transformers/tend/repository.go | 10 ++++++++-- pkg/transformers/tend/repository_test.go | 6 ++++++ pkg/transformers/test_data/wrong_model.go | 17 ++++++++++++++++ pkg/transformers/vat_init/repository.go | 10 ++++++++-- pkg/transformers/vat_init/repository_test.go | 6 ++++++ pkg/transformers/vat_move/repository.go | 9 +++++++-- pkg/transformers/vat_move/repository_test.go | 7 ++++++- 26 files changed, 210 insertions(+), 43 deletions(-) create mode 100644 pkg/transformers/test_data/wrong_model.go diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 78a741d7..64480ba3 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -15,6 +15,7 @@ package ilk import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac return err } - var ilk DripFileIlkModel for _, model := range models { - ilk = model.(DripFileIlkModel) + ilk, ok := model.(DripFileIlkModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, DripFileIlkModel{}) + } + _, err = tx.Exec( `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index 6ea5b9e1..fb1a45aa 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -89,6 +89,12 @@ var _ = Describe("Drip file ilk repository", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = dripFileIlkRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 80a6b21e..03740b40 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -15,6 +15,7 @@ package repo import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa return err } - var repo DripFileRepoModel for _, model := range models { - repo = model.(DripFileRepoModel) + repo, ok := model.(DripFileRepoModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, DripFileRepoModel{}) + } + _, err = tx.Exec( `INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 91237f94..126486c1 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -90,6 +90,12 @@ var _ = Describe("Drip file repo repository", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = dripFileRepoRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 4d3fdaa3..8d9cc0ef 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -15,6 +15,7 @@ package vow import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac return err } - var vow DripFileVowModel for _, model := range models { - vow = model.(DripFileVowModel) + vow, ok := model.(DripFileVowModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, DripFileVowModel{}) + } + _, err = tx.Exec( `INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6)`, diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index 42200cf8..70ddeb6e 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -90,6 +90,12 @@ var _ = Describe("Drip file vow repository", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = dripFileVowRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/integration_tests/drip_file_vow.go b/pkg/transformers/integration_tests/drip_file_vow.go index 7a849410..eacf35a8 100644 --- a/pkg/transformers/integration_tests/drip_file_vow.go +++ b/pkg/transformers/integration_tests/drip_file_vow.go @@ -17,8 +17,9 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/drip_file/vow" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -26,7 +27,7 @@ import ( var _ = Describe("Drip File Vow Transformer", func() { It("transforms DripFileVow log events", func() { blockNumber := int64(8762197) - config := drip_file.DripFileConfig + config := vow.DripFileVowConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber @@ -41,8 +42,13 @@ var _ = Describe("Drip File Vow Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := vow.DripFileVowTransformerInitializer{Config: config} - transformer := initializer.NewDripFileVowTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &vow.DripFileVowConverter{}, + Repository: &vow.DripFileVowRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go index 6c7bd92b..711e8857 100644 --- a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go +++ b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go @@ -17,8 +17,9 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/debt_ceiling" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -26,7 +27,7 @@ import ( var _ = Describe("PitFileDebtCeiling Transformer", func() { It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() { blockNumber := int64(8535578) - config := pit_file.PitFileConfig + config := debt_ceiling.DebtCeilingFileConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber @@ -41,8 +42,13 @@ var _ = Describe("PitFileDebtCeiling Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := debt_ceiling.PitFileDebtCeilingTransformerInitializer{Config: config} - transformer := initializer.NewPitFileDebtCeilingTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, + Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index 3c820c10..c9726b35 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -17,8 +17,9 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -26,7 +27,7 @@ import ( var _ = Describe("PitFileIlk Transformer", func() { It("fetches and transforms a PitFileIlk event from Kovan chain", func() { blockNumber := int64(9103223) - config := pit_file.PitFileConfig + config := ilk.IlkFileConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber @@ -41,8 +42,13 @@ var _ = Describe("PitFileIlk Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := ilk.PitFileIlkTransformerInitializer{Config: config} - transformer := initializer.NewPitFileIlkTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &ilk.PitFileIlkConverter{}, + Repository: &ilk.PitFileIlkRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/pit_file_stability_fee.go b/pkg/transformers/integration_tests/pit_file_stability_fee.go index 39220585..2c1f84ff 100644 --- a/pkg/transformers/integration_tests/pit_file_stability_fee.go +++ b/pkg/transformers/integration_tests/pit_file_stability_fee.go @@ -17,8 +17,9 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -26,7 +27,7 @@ import ( var _ = Describe("PitFileStabilityFee Transformer", func() { It("fetches and transforms a PitFileStabilityFee event from Kovan chain", func() { blockNumber := int64(8535544) - config := pit_file.PitFileConfig + config := stability_fee.StabilityFeeFileConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber @@ -41,8 +42,13 @@ var _ = Describe("PitFileStabilityFee Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := stability_fee.PitFileStabilityFeeTransformerInitializer{Config: config} - transformer := initializer.NewPitFileStabilityFeeTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &stability_fee.PitFileStabilityFeeConverter{}, + Repository: &stability_fee.PitFileStabilityFeeRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index bacaed03..23b49558 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -17,6 +17,8 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/tend" "github.com/vulcanize/vulcanizedb/test_config" @@ -40,8 +42,13 @@ var _ = Describe("Tend Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := tend.TendTransformerInitializer{Config: config} - transformer := initializer.NewTendTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &tend.TendConverter{}, + Repository: &tend.TendRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) @@ -74,8 +81,13 @@ var _ = Describe("Tend Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := tend.TendTransformerInitializer{Config: config} - transformer := initializer.NewTendTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &tend.TendConverter{}, + Repository: &tend.TendRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index d1b78931..fba6fb72 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -17,6 +17,8 @@ package integration_tests import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_init" "github.com/vulcanize/vulcanizedb/test_config" @@ -40,8 +42,13 @@ var _ = Describe("VatInit Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := vat_init.VatInitTransformerInitializer{Config: config} - transformer := initializer.NewVatInitTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &vat_init.VatInitConverter{}, + Repository: &vat_init.VatInitRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/vat_move.go b/pkg/transformers/integration_tests/vat_move.go index 7f93985e..0992324b 100644 --- a/pkg/transformers/integration_tests/vat_move.go +++ b/pkg/transformers/integration_tests/vat_move.go @@ -18,6 +18,8 @@ import ( "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_move" "github.com/vulcanize/vulcanizedb/test_config" @@ -41,8 +43,13 @@ var _ = Describe("VatMove Transformer", func() { err = persistHeader(rpcClient, db, blockNumber) Expect(err).NotTo(HaveOccurred()) - initializer := vat_move.VatMoveTransformerInitializer{Config: config} - transformer := initializer.NewVatMoveTransformer(db, blockchain) + initializer := factories.Transformer{ + Config: config, + Fetcher: &shared.Fetcher{}, + Converter: &vat_move.VatMoveConverter{}, + Repository: &vat_move.VatMoveRepository{}, + } + transformer := initializer.NewTransformer(db, blockchain) err = transformer.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 831ce12c..d0d28dd6 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -15,6 +15,7 @@ package debt_ceiling import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i return err } - var pitFileDC PitFileDebtCeilingModel for _, model := range models { - pitFileDC = model.(PitFileDebtCeilingModel) + pitFileDC, ok := model.(PitFileDebtCeilingModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, PitFileDebtCeilingModel{}) + } + _, err = tx.Exec( `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3::NUMERIC, $4, $5)`, diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index ab19f0a2..86f5149f 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -89,6 +89,12 @@ var _ = Describe("Pit file debt ceiling repository", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = pitFileDebtCeilingRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 7dc46138..d7f19eb6 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -15,6 +15,7 @@ package ilk import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface return err } - var pitFileIlk PitFileIlkModel for _, model := range models { - pitFileIlk = model.(PitFileIlkModel) + pitFileIlk, ok := model.(PitFileIlkModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, PitFileIlkModel{}) + } + _, err = tx.Exec( `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4::NUMERIC, $5, $6)`, diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index fc40803a..7794f1ce 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -90,6 +90,12 @@ var _ = Describe("Pit file ilk repository", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = pitFileRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/pit_file/stability_fee/repository.go b/pkg/transformers/pit_file/stability_fee/repository.go index e94b6a5e..2b5d1ce0 100644 --- a/pkg/transformers/pit_file/stability_fee/repository.go +++ b/pkg/transformers/pit_file/stability_fee/repository.go @@ -15,6 +15,7 @@ package stability_fee import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository PitFileStabilityFeeRepository) Create(headerID int64, models [] return err } - var pitFileSF PitFileStabilityFeeModel for _, model := range models { - pitFileSF = model.(PitFileStabilityFeeModel) + pitFileSF, ok := model.(PitFileStabilityFeeModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, PitFileStabilityFeeModel{}) + } + _, err = tx.Exec( `INSERT into maker.pit_file_stability_fee (header_id, what, data, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5)`, diff --git a/pkg/transformers/pit_file/stability_fee/repository_test.go b/pkg/transformers/pit_file/stability_fee/repository_test.go index 76081079..0c6fd378 100644 --- a/pkg/transformers/pit_file/stability_fee/repository_test.go +++ b/pkg/transformers/pit_file/stability_fee/repository_test.go @@ -89,6 +89,12 @@ var _ = Describe("Pit file stability fee repository", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = pitFileStabilityFeeRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 19166a84..a02cb740 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -15,6 +15,7 @@ package tend import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -28,9 +29,14 @@ func (repository TendRepository) Create(headerId int64, models []interface{}) er if err != nil { return err } - var tend TendModel + for _, model := range models { - tend = model.(TendModel) + tend, ok := model.(TendModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, TendModel{}) + } + _, err = tx.Exec( `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, tx_idx, raw_log) VALUES($1, $2, $3, $4, $5, $6, $7, $8)`, diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index 86c92e88..1cdc9953 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -98,6 +98,12 @@ var _ = Describe("TendRepository", func() { Expect(err).NotTo(HaveOccurred()) Expect(count).To(Equal(0)) }) + + It("Returns an error if model is of wrong type", func() { + err = tendRepository.Create(headerId, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/test_data/wrong_model.go b/pkg/transformers/test_data/wrong_model.go new file mode 100644 index 00000000..607cad52 --- /dev/null +++ b/pkg/transformers/test_data/wrong_model.go @@ -0,0 +1,17 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test_data + +type WrongModel struct{} diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index 815593a7..010e0a7c 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -15,6 +15,7 @@ package vat_init import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "log" @@ -29,9 +30,14 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) if err != nil { return err } - var vatInit VatInitModel + for _, model := range models { - vatInit = model.(VatInitModel) + vatInit, ok := model.(VatInitModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, VatInitModel{}) + } + log.Printf("VatInit model: %v", vatInit) _, err = tx.Exec( `INSERT INTO maker.vat_init (header_id, ilk, tx_idx, raw_log) diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index fad81598..a755a8e0 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -87,6 +87,12 @@ var _ = Describe("Vat init repository", func() { Expect(err).NotTo(HaveOccurred()) Expect(headerChecked).To(BeTrue()) }) + + It("Returns an error if model is of wrong type", func() { + err = vatInitRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MarkHeaderChecked", func() { diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index 527936ab..7d63ef25 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -15,6 +15,7 @@ package vat_move import ( + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -29,9 +30,13 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) return err } - var vatMove VatMoveModel for _, model := range models { - vatMove = model.(VatMoveModel) + vatMove, ok := model.(VatMoveModel) + if !ok { + tx.Rollback() + return fmt.Errorf("model of type %T, not %T", model, VatMoveModel{}) + } + _, err = tx.Exec( `INSERT INTO maker.vat_move (header_id, src, dst, rad, tx_idx, raw_log) VALUES ($1, $2, $3, $4::NUMERIC, $5, $6)`, diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index 986cc4d1..8b6e33ff 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -16,7 +16,6 @@ package vat_move_test import ( "database/sql" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -85,6 +84,12 @@ var _ = Describe("Vat Move", func() { Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(sql.ErrNoRows)) }) + + It("Returns an error if model is of wrong type", func() { + err = vatMoveRepository.Create(headerID, []interface{}{test_data.WrongModel{}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("model of type")) + }) }) Describe("MissingHeaders", func() { From 7ee3329bafb385e858c5d58325f01e86f401eb01 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 22 Oct 2018 15:03:41 +0200 Subject: [PATCH 32/36] Privatise DB repository field and add transformer test --- pkg/transformers/drip_file/ilk/repository.go | 12 ++++++------ .../drip_file/ilk/repository_test.go | 16 ++++------------ .../drip_file/ilk/transformer_test.go | 7 ++++++- pkg/transformers/drip_file/repo/repository.go | 12 ++++++------ .../drip_file/repo/repository_test.go | 16 ++++------------ .../drip_file/repo/transformer_test.go | 7 ++++++- pkg/transformers/drip_file/vow/repository.go | 12 ++++++------ .../drip_file/vow/repository_test.go | 16 ++++------------ .../drip_file/vow/transformer_test.go | 7 ++++++- .../pit_file/debt_ceiling/repository.go | 12 ++++++------ .../pit_file/debt_ceiling/repository_test.go | 13 +++++++------ .../pit_file/debt_ceiling/transformer_test.go | 7 ++++++- pkg/transformers/pit_file/ilk/repository.go | 12 ++++++------ pkg/transformers/pit_file/ilk/repository_test.go | 6 ++++-- .../pit_file/ilk/transformer_test.go | 5 +++++ .../pit_file/stability_fee/repository.go | 12 ++++++------ .../pit_file/stability_fee/repository_test.go | 6 ++++-- .../pit_file/stability_fee/transformer_test.go | 5 +++++ pkg/transformers/tend/repository.go | 12 ++++++------ pkg/transformers/tend/repository_test.go | 6 ++++-- pkg/transformers/tend/transformer_test.go | 5 +++++ .../test_data/mocks/drip_file/ilk/repository.go | 5 ++++- .../test_data/mocks/drip_file/repo/repository.go | 5 ++++- .../test_data/mocks/drip_file/vow/repository.go | 5 ++++- pkg/transformers/test_data/mocks/log_fetcher.go | 5 ++++- .../mocks/pit_file/debt_ceiling/repository.go | 5 ++++- .../test_data/mocks/pit_file/ilk/repository.go | 5 ++++- .../mocks/pit_file/stability_fee/repository.go | 5 ++++- .../test_data/mocks/tend/repository.go | 5 ++++- .../test_data/mocks/vat_init/repository.go | 5 ++++- .../test_data/mocks/vat_move/repository.go | 5 ++++- pkg/transformers/vat_init/repository.go | 12 ++++++------ pkg/transformers/vat_init/repository_test.go | 15 ++++----------- pkg/transformers/vat_init/transformer_test.go | 5 +++++ pkg/transformers/vat_move/repository.go | 12 ++++++------ pkg/transformers/vat_move/repository_test.go | 15 ++++----------- pkg/transformers/vat_move/transformer_test.go | 5 +++++ 37 files changed, 182 insertions(+), 138 deletions(-) diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 64480ba3..bde2d152 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -21,11 +21,11 @@ import ( ) type DripFileIlkRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository DripFileIlkRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -61,7 +61,7 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac } func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_ilk_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET drip_file_ilk_checked = $2`, headerID, true) @@ -70,7 +70,7 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -80,11 +80,11 @@ func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endi AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index fb1a45aa..80245b95 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -40,7 +40,8 @@ var _ = Describe("Drip file ilk repository", func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - dripFileIlkRepository = ilk.DripFileIlkRepository{DB: db} + dripFileIlkRepository = ilk.DripFileIlkRepository{} + dripFileIlkRepository.SetDB(db) }) Describe("Create", func() { @@ -182,7 +183,8 @@ var _ = Describe("Drip file ilk repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - dripFileIlkRepositoryTwo := ilk.DripFileIlkRepository{DB: dbTwo} + dripFileIlkRepositoryTwo := ilk.DripFileIlkRepository{} + dripFileIlkRepositoryTwo.SetDB(dbTwo) err := dripFileIlkRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -195,14 +197,4 @@ var _ = Describe("Drip file ilk repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) - - Describe("SetDB", func() { - It("sets the repository db", func() { - db := test_config.NewTestDB(core.Node{}) - repository := ilk.DripFileIlkRepository{} - Expect(repository.DB).To(BeNil()) - repository.SetDB(db) - Expect(repository.DB).To(Equal(db)) - }) - }) }) diff --git a/pkg/transformers/drip_file/ilk/transformer_test.go b/pkg/transformers/drip_file/ilk/transformer_test.go index 8089fb67..f6547bd8 100644 --- a/pkg/transformers/drip_file/ilk/transformer_test.go +++ b/pkg/transformers/drip_file/ilk/transformer_test.go @@ -50,11 +50,16 @@ var _ = Describe("Drip file ilk transformer", func() { Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 03740b40..e7e03cb1 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -21,11 +21,11 @@ import ( ) type DripFileRepoRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository DripFileRepoRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -63,7 +63,7 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa } func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_repo_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET drip_file_repo_checked = $2`, headerID, true) @@ -72,7 +72,7 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -82,11 +82,11 @@ func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, end AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 126486c1..6b690c3d 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -42,7 +42,8 @@ var _ = Describe("Drip file repo repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - dripFileRepoRepository = repo.DripFileRepoRepository{DB: db} + dripFileRepoRepository = repo.DripFileRepoRepository{} + dripFileRepoRepository.SetDB(db) }) Describe("Create", func() { @@ -182,7 +183,8 @@ var _ = Describe("Drip file repo repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - dripFileRepoRepositoryTwo := repo.DripFileRepoRepository{DB: dbTwo} + dripFileRepoRepositoryTwo := repo.DripFileRepoRepository{} + dripFileRepoRepositoryTwo.SetDB(dbTwo) err := dripFileRepoRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -195,14 +197,4 @@ var _ = Describe("Drip file repo repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) - - Describe("SetDB", func() { - It("sets the repository db", func() { - db := test_config.NewTestDB(core.Node{}) - repository := repo.DripFileRepoRepository{} - Expect(repository.DB).To(BeNil()) - repository.SetDB(db) - Expect(repository.DB).To(Equal(db)) - }) - }) }) diff --git a/pkg/transformers/drip_file/repo/transformer_test.go b/pkg/transformers/drip_file/repo/transformer_test.go index a98a06bf..14a16475 100644 --- a/pkg/transformers/drip_file/repo/transformer_test.go +++ b/pkg/transformers/drip_file/repo/transformer_test.go @@ -50,11 +50,16 @@ var _ = Describe("Drip file repo transformer", func() { Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 8d9cc0ef..32a4140a 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -21,11 +21,11 @@ import ( ) type DripFileVowRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository DripFileVowRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -62,7 +62,7 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac } func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, drip_file_vow_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET drip_file_vow_checked = $2`, headerID, true) @@ -71,7 +71,7 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -81,11 +81,11 @@ func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endi AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index 70ddeb6e..0eb48545 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -42,7 +42,8 @@ var _ = Describe("Drip file vow repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - dripFileVowRepository = vow.DripFileVowRepository{DB: db} + dripFileVowRepository = vow.DripFileVowRepository{} + dripFileVowRepository.SetDB(db) }) Describe("Create", func() { @@ -182,7 +183,8 @@ var _ = Describe("Drip file vow repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - dripFileVowRepositoryTwo := vow.DripFileVowRepository{DB: dbTwo} + dripFileVowRepositoryTwo := vow.DripFileVowRepository{} + dripFileVowRepositoryTwo.SetDB(dbTwo) err := dripFileVowRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -195,14 +197,4 @@ var _ = Describe("Drip file vow repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) - - Describe("SetDB", func() { - It("sets the repository db", func() { - db := test_config.NewTestDB(core.Node{}) - repository := vow.DripFileVowRepository{} - Expect(repository.DB).To(BeNil()) - repository.SetDB(db) - Expect(repository.DB).To(Equal(db)) - }) - }) }) diff --git a/pkg/transformers/drip_file/vow/transformer_test.go b/pkg/transformers/drip_file/vow/transformer_test.go index 4d9e7d23..40b92736 100644 --- a/pkg/transformers/drip_file/vow/transformer_test.go +++ b/pkg/transformers/drip_file/vow/transformer_test.go @@ -50,11 +50,16 @@ var _ = Describe("Drip file vow transformer", func() { Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index d0d28dd6..14b59c16 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -21,11 +21,11 @@ import ( ) type PitFileDebtCeilingRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -63,7 +63,7 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i } func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_debt_ceiling_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET pit_file_debt_ceiling_checked = $2`, headerID, true) @@ -72,7 +72,7 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -82,11 +82,11 @@ func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumbe AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index 86f5149f..33dcaf95 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -42,7 +42,8 @@ var _ = Describe("Pit file debt ceiling repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - pitFileDebtCeilingRepository = debt_ceiling.PitFileDebtCeilingRepository{DB: db} + pitFileDebtCeilingRepository = debt_ceiling.PitFileDebtCeilingRepository{} + pitFileDebtCeilingRepository.SetDB(db) }) Describe("Create", func() { @@ -182,16 +183,16 @@ var _ = Describe("Pit file debt ceiling repository", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - pitFileRepository := debt_ceiling.PitFileDebtCeilingRepository{DB: db} - pitFileRepositoryTwo := debt_ceiling.PitFileDebtCeilingRepository{DB: dbTwo} - err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) + pitFileDebtCeilingRepositoryTwo := debt_ceiling.PitFileDebtCeilingRepository{} + pitFileDebtCeilingRepositoryTwo.SetDB(dbTwo) + err := pitFileDebtCeilingRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := pitFileRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeOneMissingHeaders, err := pitFileDebtCeilingRepository.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeOneMissingHeaders)).To(Equal(len(blockNumbers) - 1)) - nodeTwoMissingHeaders, err := pitFileRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) + nodeTwoMissingHeaders, err := pitFileDebtCeilingRepositoryTwo.MissingHeaders(blockNumbers[0], blockNumbers[len(blockNumbers)-1]) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) diff --git a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go index 96c9d9d1..59938c18 100644 --- a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go @@ -52,7 +52,12 @@ var _ = Describe("Pit file debt ceiling transformer", func() { Fetcher: &fetcher, Converter: &converter, Repository: &repository, - } + }.NewTransformer(nil, nil) + }) + + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) }) It("gets missing headers for block numbers specified in config", func() { diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index d7f19eb6..79cc6576 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -21,11 +21,11 @@ import ( ) type PitFileIlkRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository PitFileIlkRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -59,7 +59,7 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface } func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_ilk_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET pit_file_ilk_checked = $2`, headerID, true) @@ -68,7 +68,7 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -78,11 +78,11 @@ func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endin AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index 7794f1ce..9b852970 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -42,7 +42,8 @@ var _ = Describe("Pit file ilk repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - pitFileRepository = ilk.PitFileIlkRepository{DB: db} + pitFileRepository = ilk.PitFileIlkRepository{} + pitFileRepository.SetDB(db) }) Describe("Create", func() { @@ -183,7 +184,8 @@ var _ = Describe("Pit file ilk repository", func() { Expect(err).NotTo(HaveOccurred()) } - pitFileRepositoryTwo := ilk.PitFileIlkRepository{DB: dbTwo} + pitFileRepositoryTwo := ilk.PitFileIlkRepository{} + pitFileRepositoryTwo.SetDB(dbTwo) err := pitFileRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/pit_file/ilk/transformer_test.go b/pkg/transformers/pit_file/ilk/transformer_test.go index 89474d55..b98b712a 100644 --- a/pkg/transformers/pit_file/ilk/transformer_test.go +++ b/pkg/transformers/pit_file/ilk/transformer_test.go @@ -55,6 +55,11 @@ var _ = Describe("Pit file ilk transformer", func() { }.NewTransformer(nil, nil) }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() diff --git a/pkg/transformers/pit_file/stability_fee/repository.go b/pkg/transformers/pit_file/stability_fee/repository.go index 2b5d1ce0..f0421175 100644 --- a/pkg/transformers/pit_file/stability_fee/repository.go +++ b/pkg/transformers/pit_file/stability_fee/repository.go @@ -21,11 +21,11 @@ import ( ) type PitFileStabilityFeeRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository PitFileStabilityFeeRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -63,7 +63,7 @@ func (repository PitFileStabilityFeeRepository) Create(headerID int64, models [] } func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, pit_file_stability_fee_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET pit_file_stability_fee_checked = $2`, headerID, true) @@ -72,7 +72,7 @@ func (repository PitFileStabilityFeeRepository) MarkHeaderChecked(headerID int64 func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -82,11 +82,11 @@ func (repository PitFileStabilityFeeRepository) MissingHeaders(startingBlockNumb AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *PitFileStabilityFeeRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/pit_file/stability_fee/repository_test.go b/pkg/transformers/pit_file/stability_fee/repository_test.go index 0c6fd378..068d5e74 100644 --- a/pkg/transformers/pit_file/stability_fee/repository_test.go +++ b/pkg/transformers/pit_file/stability_fee/repository_test.go @@ -42,7 +42,8 @@ var _ = Describe("Pit file stability fee repository", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - pitFileStabilityFeeRepository = stability_fee.PitFileStabilityFeeRepository{DB: db} + pitFileStabilityFeeRepository = stability_fee.PitFileStabilityFeeRepository{} + pitFileStabilityFeeRepository.SetDB(db) }) Describe("Create", func() { @@ -182,7 +183,8 @@ var _ = Describe("Pit file stability fee repository", func() { Expect(err).NotTo(HaveOccurred()) } - pitFileRepositoryTwo := stability_fee.PitFileStabilityFeeRepository{DB: dbTwo} + pitFileRepositoryTwo := stability_fee.PitFileStabilityFeeRepository{} + pitFileRepositoryTwo.SetDB(dbTwo) err = pitFileStabilityFeeRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/pit_file/stability_fee/transformer_test.go b/pkg/transformers/pit_file/stability_fee/transformer_test.go index 633d7feb..17128054 100644 --- a/pkg/transformers/pit_file/stability_fee/transformer_test.go +++ b/pkg/transformers/pit_file/stability_fee/transformer_test.go @@ -55,6 +55,11 @@ var _ = Describe("Pit file stability fee transformer", func() { headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index a02cb740..de1ef581 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -21,11 +21,11 @@ import ( ) type TendRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository TendRepository) Create(headerId int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -61,7 +61,7 @@ func (repository TendRepository) Create(headerId int64, models []interface{}) er } func (repository TendRepository) MarkHeaderChecked(headerId int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, tend_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, tend_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET tend_checked = $2`, headerId, true) @@ -70,7 +70,7 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error { func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -80,12 +80,12 @@ func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlock AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *TendRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index 1cdc9953..e43e2d5e 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -41,7 +41,8 @@ var _ = Describe("TendRepository", func() { db = test_config.NewTestDB(node) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - tendRepository = tend.TendRepository{DB: db} + tendRepository = tend.TendRepository{} + tendRepository.SetDB(db) }) Describe("Create", func() { @@ -187,7 +188,8 @@ var _ = Describe("TendRepository", func() { node2 := core.Node{} db2 := test_config.NewTestDB(node2) headerRepository2 := repositories.NewHeaderRepository(db2) - tendRepository2 := tend.TendRepository{DB: db2} + tendRepository2 := tend.TendRepository{} + tendRepository2.SetDB(db2) for _, number := range []int64{startingBlock, tendBlock, endingBlock} { headerRepository2.CreateOrUpdateHeader(fakes.GetFakeHeader(number)) diff --git a/pkg/transformers/tend/transformer_test.go b/pkg/transformers/tend/transformer_test.go index 271f040b..ff8b3ed0 100644 --- a/pkg/transformers/tend/transformer_test.go +++ b/pkg/transformers/tend/transformer_test.go @@ -56,6 +56,11 @@ var _ = Describe("Tend Transformer", func() { }.NewTransformer(nil, nil) }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for blocks in the configured range", func() { err := transformer.Execute() diff --git a/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go b/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go index fe96b361..3f99a840 100644 --- a/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/ilk/repository.go @@ -31,6 +31,7 @@ type MockDripFileIlkRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedModels []interface{} + SetDbCalled bool } func (repository *MockDripFileIlkRepository) Create(headerID int64, models []interface{}) error { @@ -69,4 +70,6 @@ func (repository *MockDripFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockDripFileIlkRepository) SetDB(db *postgres.DB) {} +func (repository *MockDripFileIlkRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/drip_file/repo/repository.go b/pkg/transformers/test_data/mocks/drip_file/repo/repository.go index 40131fff..b1ae32c3 100644 --- a/pkg/transformers/test_data/mocks/drip_file/repo/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/repo/repository.go @@ -31,6 +31,7 @@ type MockDripFileRepoRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedModels []interface{} + SetDbCalled bool } func (repository *MockDripFileRepoRepository) Create(headerID int64, models []interface{}) error { @@ -70,4 +71,6 @@ func (repository *MockDripFileRepoRepository) AssertMarkHeaderCheckedCalledWith( Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockDripFileRepoRepository) SetDB(db *postgres.DB) {} +func (repository *MockDripFileRepoRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/drip_file/vow/repository.go b/pkg/transformers/test_data/mocks/drip_file/vow/repository.go index e365817a..2487ca42 100644 --- a/pkg/transformers/test_data/mocks/drip_file/vow/repository.go +++ b/pkg/transformers/test_data/mocks/drip_file/vow/repository.go @@ -31,6 +31,7 @@ type MockDripFileVowRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedModels []interface{} + SetDbCalled bool } func (repository *MockDripFileVowRepository) Create(headerID int64, models []interface{}) error { @@ -69,4 +70,6 @@ func (repository *MockDripFileVowRepository) AssertMarkHeaderCheckedCalledWith(i Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockDripFileVowRepository) SetDB(db *postgres.DB) {} +func (repository *MockDripFileVowRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/log_fetcher.go b/pkg/transformers/test_data/mocks/log_fetcher.go index b6b591a7..53172207 100644 --- a/pkg/transformers/test_data/mocks/log_fetcher.go +++ b/pkg/transformers/test_data/mocks/log_fetcher.go @@ -26,6 +26,7 @@ type MockLogFetcher struct { FetchedBlocks []int64 fetcherError error FetchedLogs []types.Log + SetBcCalled bool } func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) { @@ -36,7 +37,9 @@ func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]comm return mlf.FetchedLogs, mlf.fetcherError } -func (mlf *MockLogFetcher) SetBC(_ core.BlockChain) {} +func (mlf *MockLogFetcher) SetBC(bc core.BlockChain) { + mlf.SetBcCalled = true +} func (mlf *MockLogFetcher) SetFetcherError(err error) { mlf.fetcherError = err diff --git a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go index 4ba8ec08..7c9da60f 100644 --- a/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/debt_ceiling/repository.go @@ -31,6 +31,7 @@ type MockPitFileDebtCeilingRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedModels []interface{} + SetDbCalled bool } func (repository *MockPitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { @@ -70,4 +71,6 @@ func (repository *MockPitFileDebtCeilingRepository) AssertMarkHeaderCheckedCalle Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockPitFileDebtCeilingRepository) SetDB(db *postgres.DB) {} +func (repository *MockPitFileDebtCeilingRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go index 0ab6fc75..311b01a5 100644 --- a/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/ilk/repository.go @@ -31,6 +31,7 @@ type MockPitFileIlkRepository struct { markHeaderCheckedPassedHeaderID int64 missingHeaders []core.Header missingHeadersError error + SetDbCalled bool } func (repository *MockPitFileIlkRepository) SetCreateError(err error) { @@ -70,4 +71,6 @@ func (repository *MockPitFileIlkRepository) AssertMarkHeaderCheckedCalledWith(i Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockPitFileIlkRepository) SetDB(db *postgres.DB) {} +func (repository *MockPitFileIlkRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go b/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go index 53967bd0..e20678e1 100644 --- a/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go +++ b/pkg/transformers/test_data/mocks/pit_file/stability_fee/repository.go @@ -31,6 +31,7 @@ type MockPitFileStabilityFeeRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedModels []interface{} + SetDbCalled bool } func (repository *MockPitFileStabilityFeeRepository) Create(headerID int64, models []interface{}) error { @@ -70,4 +71,6 @@ func (repository *MockPitFileStabilityFeeRepository) AssertMarkHeaderCheckedCall Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockPitFileStabilityFeeRepository) SetDB(db *postgres.DB) {} +func (repository *MockPitFileStabilityFeeRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/tend/repository.go b/pkg/transformers/test_data/mocks/tend/repository.go index cfb6e9af..b013b9ab 100644 --- a/pkg/transformers/test_data/mocks/tend/repository.go +++ b/pkg/transformers/test_data/mocks/tend/repository.go @@ -31,6 +31,7 @@ type MockTendRepository struct { markHeaderCheckedPassedHeaderId int64 missingHeaders []core.Header missingHeadersError error + SetDbCalled bool } func (repository *MockTendRepository) Create(headerId int64, tend []interface{}) error { @@ -70,4 +71,6 @@ func (repository *MockTendRepository) AssertMarkHeaderCheckedCalledWith(headerId Expect(repository.markHeaderCheckedPassedHeaderId).To(Equal(headerId)) } -func (repository *MockTendRepository) SetDB(db *postgres.DB) {} +func (repository *MockTendRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/vat_init/repository.go b/pkg/transformers/test_data/mocks/vat_init/repository.go index 298c941d..fccbdc0c 100644 --- a/pkg/transformers/test_data/mocks/vat_init/repository.go +++ b/pkg/transformers/test_data/mocks/vat_init/repository.go @@ -31,6 +31,7 @@ type MockVatInitRepository struct { PassedEndingBlockNumber int64 PassedHeaderID int64 PassedModels []interface{} + SetDbCalled bool } func (repository *MockVatInitRepository) Create(headerID int64, models []interface{}) error { @@ -70,4 +71,6 @@ func (repository *MockVatInitRepository) AssertMarkHeaderCheckedCalledWith(i int Expect(repository.markHeaderCheckedPassedHeaderID).To(Equal(i)) } -func (repository *MockVatInitRepository) SetDB(db *postgres.DB) {} +func (repository *MockVatInitRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/test_data/mocks/vat_move/repository.go b/pkg/transformers/test_data/mocks/vat_move/repository.go index 46c3eb02..d1ce0859 100644 --- a/pkg/transformers/test_data/mocks/vat_move/repository.go +++ b/pkg/transformers/test_data/mocks/vat_move/repository.go @@ -29,6 +29,7 @@ type MockVatMoveRepository struct { PassedModels []interface{} CheckedHeaderIDs []int64 CheckedHeaderError error + SetDbCalled bool } func (repository *MockVatMoveRepository) Create(headerID int64, models []interface{}) error { @@ -64,4 +65,6 @@ func (repository *MockVatMoveRepository) SetCheckedHeaderError(e error) { repository.CheckedHeaderError = e } -func (repository *MockVatMoveRepository) SetDB(db *postgres.DB) {} +func (repository *MockVatMoveRepository) SetDB(db *postgres.DB) { + repository.SetDbCalled = true +} diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index 010e0a7c..b581b8b1 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -22,11 +22,11 @@ import ( ) type VatInitRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository VatInitRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -64,7 +64,7 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) } func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_init_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET vat_init_checked = $2`, headerID, true) @@ -73,7 +73,7 @@ func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -83,11 +83,11 @@ func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBl AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository *VatInitRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index a755a8e0..0738db18 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -40,7 +40,8 @@ var _ = Describe("Vat init repository", func() { BeforeEach(func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) - vatInitRepository = vat_init.VatInitRepository{DB: db} + vatInitRepository = vat_init.VatInitRepository{} + vatInitRepository.SetDB(db) headerRepository = repositories.NewHeaderRepository(db) }) @@ -180,7 +181,8 @@ var _ = Describe("Vat init repository", func() { Expect(err).NotTo(HaveOccurred()) } - vatInitRepositoryTwo := vat_init.VatInitRepository{DB: dbTwo} + vatInitRepositoryTwo := vat_init.VatInitRepository{} + vatInitRepositoryTwo.SetDB(dbTwo) err = vatInitRepository.MarkHeaderChecked(headerIDs[0]) Expect(err).NotTo(HaveOccurred()) @@ -193,13 +195,4 @@ var _ = Describe("Vat init repository", func() { Expect(len(nodeTwoMissingHeaders)).To(Equal(len(blockNumbers))) }) }) - - Describe("SetDB", func() { - It("sets the repository db", func() { - vatInitRepository.SetDB(nil) - Expect(vatInitRepository.DB).To(BeNil()) - vatInitRepository.SetDB(db) - Expect(vatInitRepository.DB).To(Equal(db)) - }) - }) }) diff --git a/pkg/transformers/vat_init/transformer_test.go b/pkg/transformers/vat_init/transformer_test.go index d80a1334..406c17a0 100644 --- a/pkg/transformers/vat_init/transformer_test.go +++ b/pkg/transformers/vat_init/transformer_test.go @@ -55,6 +55,11 @@ var _ = Describe("Vat init transformer", func() { }.NewTransformer(nil, nil) }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index 7d63ef25..91928ae4 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -21,11 +21,11 @@ import ( ) type VatMoveRepository struct { - DB *postgres.DB + db *postgres.DB } func (repository VatMoveRepository) Create(headerID int64, models []interface{}) error { - tx, err := repository.DB.Begin() + tx, err := repository.db.Begin() if err != nil { return err } @@ -66,7 +66,7 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { var result []core.Header - err := repository.DB.Select( + err := repository.db.Select( &result, `SELECT headers.id, headers.block_number FROM headers LEFT JOIN checked_headers on headers.id = header_id @@ -76,14 +76,14 @@ func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBl AND headers.eth_node_fingerprint = $3`, startingBlockNumber, endingBlockNumber, - repository.DB.Node.ID, + repository.db.Node.ID, ) return result, err } func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { - _, err := repository.DB.Exec(`INSERT INTO public.checked_headers (header_id, vat_move_checked) + _, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_move_checked) VALUES ($1, $2) ON CONFLICT (header_id) DO UPDATE SET vat_move_checked = $2`, headerID, true) @@ -91,5 +91,5 @@ func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { } func (repository *VatMoveRepository) SetDB(db *postgres.DB) { - repository.DB = db + repository.db = db } diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index 8b6e33ff..0e39567e 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -37,7 +37,8 @@ var _ = Describe("Vat Move", func() { db = test_config.NewTestDB(core.Node{}) test_config.CleanTestDB(db) headerRepository = repositories.NewHeaderRepository(db) - vatMoveRepository = vat_move.VatMoveRepository{DB: db} + vatMoveRepository = vat_move.VatMoveRepository{} + vatMoveRepository.SetDB(db) }) Describe("Create", func() { @@ -150,7 +151,8 @@ var _ = Describe("Vat Move", func() { _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) Expect(err).NotTo(HaveOccurred()) } - vatMoveRepositoryTwo := vat_move.VatMoveRepository{DB: dbTwo} + vatMoveRepositoryTwo := vat_move.VatMoveRepository{} + vatMoveRepositoryTwo.SetDB(dbTwo) err := vatMoveRepository.Create(headerIDs[0], []interface{}{test_data.VatMoveModel}) Expect(err).NotTo(HaveOccurred()) @@ -195,13 +197,4 @@ var _ = Describe("Vat Move", func() { Expect(headerChecked).To(BeTrue()) }) }) - - Describe("SetDB", func() { - It("sets the repository db", func() { - vatMoveRepository.DB = nil - Expect(vatMoveRepository.DB).To(BeNil()) - vatMoveRepository.SetDB(db) - Expect(vatMoveRepository.DB).To(Equal(db)) - }) - }) }) diff --git a/pkg/transformers/vat_move/transformer_test.go b/pkg/transformers/vat_move/transformer_test.go index 0e35f6d3..069c6f1d 100644 --- a/pkg/transformers/vat_move/transformer_test.go +++ b/pkg/transformers/vat_move/transformer_test.go @@ -52,6 +52,11 @@ var _ = Describe("Vat move transformer", func() { }.NewTransformer(nil, nil) }) + It("sets the blockchain and database", func() { + Expect(fetcher.SetBcCalled).To(BeTrue()) + Expect(repository.SetDbCalled).To(BeTrue()) + }) + It("gets missing headers for block numbers specified in config", func() { err := transformer.Execute() From 6d9547a888b04f221700478a1fd24b78cbbf75e1 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 22 Oct 2018 15:45:56 +0200 Subject: [PATCH 33/36] Fix invalidated test for tend repository --- pkg/transformers/tend/repository_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index e43e2d5e..304f2200 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -111,9 +111,6 @@ var _ = Describe("TendRepository", func() { BeforeEach(func() { headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - - err = tendRepository.Create(headerId, []interface{}{test_data.TendModel}) - Expect(err).NotTo(HaveOccurred()) }) It("creates a row for a new headerID", func() { From 794a7e5f99287f599018465e249cec92a985c7db Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 22 Oct 2018 15:51:05 +0200 Subject: [PATCH 34/36] Nuke some leftover converter interfaces --- pkg/transformers/drip_file/ilk/converter.go | 4 ---- pkg/transformers/drip_file/vow/converter.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/pkg/transformers/drip_file/ilk/converter.go b/pkg/transformers/drip_file/ilk/converter.go index c2ca2dec..4f03c473 100644 --- a/pkg/transformers/drip_file/ilk/converter.go +++ b/pkg/transformers/drip_file/ilk/converter.go @@ -24,10 +24,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]DripFileIlkModel, error) -} - type DripFileIlkConverter struct{} func (DripFileIlkConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { diff --git a/pkg/transformers/drip_file/vow/converter.go b/pkg/transformers/drip_file/vow/converter.go index a4d48b28..dac76194 100644 --- a/pkg/transformers/drip_file/vow/converter.go +++ b/pkg/transformers/drip_file/vow/converter.go @@ -22,10 +22,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -type Converter interface { - ToModels(ethLogs []types.Log) ([]DripFileVowModel, error) -} - type DripFileVowConverter struct{} func (DripFileVowConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) { From f55449d0e5d1f8604cd4c6f79cbdf52e3c98bb96 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 22 Oct 2018 16:14:02 +0200 Subject: [PATCH 35/36] Replace GinkgoRandomSeed with proper randomisation --- pkg/transformers/drip_file/ilk/transformer_test.go | 5 +++-- pkg/transformers/drip_file/repo/repository_test.go | 3 ++- pkg/transformers/drip_file/repo/transformer_test.go | 5 +++-- pkg/transformers/drip_file/vow/repository_test.go | 3 ++- pkg/transformers/drip_file/vow/transformer_test.go | 5 +++-- pkg/transformers/pit_file/debt_ceiling/repository_test.go | 3 ++- pkg/transformers/pit_file/debt_ceiling/transformer_test.go | 5 +++-- pkg/transformers/pit_file/ilk/repository_test.go | 3 ++- pkg/transformers/pit_file/ilk/transformer_test.go | 5 +++-- pkg/transformers/pit_file/stability_fee/repository_test.go | 3 ++- pkg/transformers/pit_file/stability_fee/transformer_test.go | 5 +++-- pkg/transformers/tend/repository_test.go | 3 ++- pkg/transformers/tend/transformer_test.go | 5 +++-- pkg/transformers/vat_init/repository_test.go | 3 ++- pkg/transformers/vat_init/transformer_test.go | 5 +++-- pkg/transformers/vat_move/repository_test.go | 3 ++- pkg/transformers/vat_move/transformer_test.go | 5 +++-- 17 files changed, 43 insertions(+), 26 deletions(-) diff --git a/pkg/transformers/drip_file/ilk/transformer_test.go b/pkg/transformers/drip_file/ilk/transformer_test.go index f6547bd8..0bba4782 100644 --- a/pkg/transformers/drip_file/ilk/transformer_test.go +++ b/pkg/transformers/drip_file/ilk/transformer_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -51,8 +52,8 @@ var _ = Describe("Drip file ilk transformer", func() { Converter: &converter, Repository: &repository, }.NewTransformer(nil, nil) - headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} - headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + headerOne = core.Header{BlockNumber: rand.Int63(), Id: rand.Int63()} + headerTwo = core.Header{BlockNumber: rand.Int63(), Id: rand.Int63()} }) It("sets the blockchain and database", func() { diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 6b690c3d..6e63f901 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -16,6 +16,7 @@ package repo_test import ( "database/sql" + "math/rand" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -137,7 +138,7 @@ var _ = Describe("Drip file repo repository", func() { ) BeforeEach(func() { - startingBlock = GinkgoRandomSeed() + startingBlock = rand.Int63() dripFileBlock = startingBlock + 1 endingBlock = startingBlock + 2 diff --git a/pkg/transformers/drip_file/repo/transformer_test.go b/pkg/transformers/drip_file/repo/transformer_test.go index 14a16475..e2dd912d 100644 --- a/pkg/transformers/drip_file/repo/transformer_test.go +++ b/pkg/transformers/drip_file/repo/transformer_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -51,8 +52,8 @@ var _ = Describe("Drip file repo transformer", func() { Converter: &converter, Repository: &repository, }.NewTransformer(nil, nil) - headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} - headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + headerOne = core.Header{BlockNumber: rand.Int63(), Id: rand.Int63()} + headerTwo = core.Header{BlockNumber: rand.Int63(), Id: rand.Int63()} }) It("sets the blockchain and database", func() { diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index 0eb48545..914f5fb6 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -16,6 +16,7 @@ package vow_test import ( "database/sql" + "math/rand" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -137,7 +138,7 @@ var _ = Describe("Drip file vow repository", func() { ) BeforeEach(func() { - startingBlock = GinkgoRandomSeed() + startingBlock = rand.Int63() dripFileBlock = startingBlock + 1 endingBlock = startingBlock + 2 diff --git a/pkg/transformers/drip_file/vow/transformer_test.go b/pkg/transformers/drip_file/vow/transformer_test.go index 40b92736..451bb6d1 100644 --- a/pkg/transformers/drip_file/vow/transformer_test.go +++ b/pkg/transformers/drip_file/vow/transformer_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -51,8 +52,8 @@ var _ = Describe("Drip file vow transformer", func() { Converter: &converter, Repository: &repository, }.NewTransformer(nil, nil) - headerOne = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} - headerTwo = core.Header{BlockNumber: GinkgoRandomSeed(), Id: GinkgoRandomSeed()} + headerOne = core.Header{BlockNumber: rand.Int63(), Id: rand.Int63()} + headerTwo = core.Header{BlockNumber: rand.Int63(), Id: rand.Int63()} }) It("sets the blockchain and database", func() { diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index 33dcaf95..23b0da01 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -16,6 +16,7 @@ package debt_ceiling_test import ( "database/sql" + "math/rand" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -136,7 +137,7 @@ var _ = Describe("Pit file debt ceiling repository", func() { ) BeforeEach(func() { - startingBlock = GinkgoRandomSeed() + startingBlock = rand.Int63() pitFileBlock = startingBlock + 1 endingBlock = startingBlock + 2 diff --git a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go index 59938c18..ede2708d 100644 --- a/pkg/transformers/pit_file/debt_ceiling/transformer_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/transformer_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -45,8 +46,8 @@ var _ = Describe("Pit file debt ceiling transformer", func() { fetcher = mocks.MockLogFetcher{} converter = debt_ceiling_mocks.MockPitFileDebtCeilingConverter{} repository = debt_ceiling_mocks.MockPitFileDebtCeilingRepository{} - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} + headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} transformer = factories.Transformer{ Config: config, Fetcher: &fetcher, diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index 9b852970..db7f127d 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -16,6 +16,7 @@ package ilk_test import ( "database/sql" + "math/rand" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -137,7 +138,7 @@ var _ = Describe("Pit file ilk repository", func() { ) BeforeEach(func() { - startingBlock = GinkgoRandomSeed() + startingBlock = rand.Int63() pitFileBlock = startingBlock + 1 endingBlock = startingBlock + 2 diff --git a/pkg/transformers/pit_file/ilk/transformer_test.go b/pkg/transformers/pit_file/ilk/transformer_test.go index b98b712a..4a19c457 100644 --- a/pkg/transformers/pit_file/ilk/transformer_test.go +++ b/pkg/transformers/pit_file/ilk/transformer_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -45,8 +46,8 @@ var _ = Describe("Pit file ilk transformer", func() { fetcher = mocks.MockLogFetcher{} converter = ilk_mocks.MockPitFileIlkConverter{} repository = ilk_mocks.MockPitFileIlkRepository{} - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} + headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} transformer = factories.Transformer{ Config: config, Converter: &converter, diff --git a/pkg/transformers/pit_file/stability_fee/repository_test.go b/pkg/transformers/pit_file/stability_fee/repository_test.go index 068d5e74..3fc01692 100644 --- a/pkg/transformers/pit_file/stability_fee/repository_test.go +++ b/pkg/transformers/pit_file/stability_fee/repository_test.go @@ -16,6 +16,7 @@ package stability_fee_test import ( "database/sql" + "math/rand" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -136,7 +137,7 @@ var _ = Describe("Pit file stability fee repository", func() { ) BeforeEach(func() { - startingBlock = GinkgoRandomSeed() + startingBlock = rand.Int63() pitFileBlock = startingBlock + 1 endingBlock = startingBlock + 2 diff --git a/pkg/transformers/pit_file/stability_fee/transformer_test.go b/pkg/transformers/pit_file/stability_fee/transformer_test.go index 17128054..ffab7ced 100644 --- a/pkg/transformers/pit_file/stability_fee/transformer_test.go +++ b/pkg/transformers/pit_file/stability_fee/transformer_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/stability_fee" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -51,8 +52,8 @@ var _ = Describe("Pit file stability fee transformer", func() { Converter: &converter, Repository: &repository, }.NewTransformer(nil, nil) - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} + headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} }) It("sets the blockchain and database", func() { diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index 304f2200..d9d2e7a3 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -17,6 +17,7 @@ package tend_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -143,7 +144,7 @@ var _ = Describe("TendRepository", func() { ) BeforeEach(func() { - tendBlock = GinkgoRandomSeed() + tendBlock = rand.Int63() startingBlock = tendBlock - 1 endingBlock = tendBlock + 1 outOfRangeBlock = tendBlock + 2 diff --git a/pkg/transformers/tend/transformer_test.go b/pkg/transformers/tend/transformer_test.go index ff8b3ed0..1f55a832 100644 --- a/pkg/transformers/tend/transformer_test.go +++ b/pkg/transformers/tend/transformer_test.go @@ -16,6 +16,7 @@ package tend_test import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" @@ -46,8 +47,8 @@ var _ = Describe("Tend Transformer", func() { converter = tend_mocks.MockTendConverter{} repository = tend_mocks.MockTendRepository{} fetcher = mocks.MockLogFetcher{} - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} + headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} transformer = factories.Transformer{ Config: config, Fetcher: &fetcher, diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index 0738db18..28d1f29d 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -16,6 +16,7 @@ package vat_init_test import ( "database/sql" + "math/rand" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -134,7 +135,7 @@ var _ = Describe("Vat init repository", func() { ) BeforeEach(func() { - startingBlock = GinkgoRandomSeed() + startingBlock = rand.Int63() vatInitBlock = startingBlock + 1 endingBlock = startingBlock + 2 diff --git a/pkg/transformers/vat_init/transformer_test.go b/pkg/transformers/vat_init/transformer_test.go index 406c17a0..9df2e1da 100644 --- a/pkg/transformers/vat_init/transformer_test.go +++ b/pkg/transformers/vat_init/transformer_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" @@ -45,8 +46,8 @@ var _ = Describe("Vat init transformer", func() { converter = vat_init_mocks.MockVatInitConverter{} repository = vat_init_mocks.MockVatInitRepository{} fetcher = mocks.MockLogFetcher{} - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} + headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} transformer = factories.Transformer{ Config: config, Fetcher: &fetcher, diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index 0e39567e..1855ea83 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -18,6 +18,7 @@ import ( "database/sql" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "math/rand" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -94,7 +95,7 @@ var _ = Describe("Vat Move", func() { }) Describe("MissingHeaders", func() { - var eventBlockNumber = GinkgoRandomSeed() + var eventBlockNumber = rand.Int63() var startingBlockNumber = eventBlockNumber - 1 var endingBlockNumber = eventBlockNumber + 1 var outOfRangeBlockNumber = eventBlockNumber + 2 diff --git a/pkg/transformers/vat_move/transformer_test.go b/pkg/transformers/vat_move/transformer_test.go index 069c6f1d..fa271e3b 100644 --- a/pkg/transformers/vat_move/transformer_test.go +++ b/pkg/transformers/vat_move/transformer_test.go @@ -27,6 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" vat_move_mocks "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks/vat_move" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_move" + "math/rand" ) var _ = Describe("Vat move transformer", func() { @@ -42,8 +43,8 @@ var _ = Describe("Vat move transformer", func() { fetcher = mocks.MockLogFetcher{} converter = vat_move_mocks.MockVatMoveConverter{} repository = vat_move_mocks.MockVatMoveRepository{} - headerOne = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} - headerTwo = core.Header{Id: GinkgoRandomSeed(), BlockNumber: GinkgoRandomSeed()} + headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} + headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} transformer = factories.Transformer{ Config: config, Converter: &converter, From a0deb5eb620421bf068c74f75614a0c748fb849e Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 23 Oct 2018 11:19:36 +0200 Subject: [PATCH 36/36] Add and use transformer labels in shared constants --- cmd/continuousLogSync.go | 54 +++++++++---------- pkg/transformers/drip_file/ilk/config.go | 2 +- pkg/transformers/drip_file/repo/config.go | 2 +- pkg/transformers/drip_file/vow/config.go | 2 +- .../pit_file/debt_ceiling/config.go | 2 +- pkg/transformers/pit_file/ilk/config.go | 2 +- .../pit_file/stability_fee/config.go | 2 +- pkg/transformers/shared/constants.go | 28 ++++++++++ pkg/transformers/tend/config.go | 2 +- pkg/transformers/vat_init/config.go | 2 +- pkg/transformers/vat_move/config.go | 2 +- 11 files changed, 64 insertions(+), 36 deletions(-) diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 6d8c66f7..dc09541e 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -86,33 +86,33 @@ func getTransformerInititalizers(transformerNames []string) []shared2.Transforme func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { transformerInitializerMap := make(map[string]shared2.TransformerInitializer) - transformerInitializerMap["bite"] = transformers.BiteTransformerInitializer - transformerInitializerMap["catFileChopLump"] = transformers.CatFileChopLumpTransformerInitializer - transformerInitializerMap["catFileFlip"] = transformers.CatFileFlipTransformerInitializer - transformerInitializerMap["catFilePitVow"] = transformers.CatFilePitVowTransformerInitializer - transformerInitializerMap["deal"] = transformers.DealTransformerInitializer - transformerInitializerMap["dent"] = transformers.DentTransformerInitializer - transformerInitializerMap["dripDrip"] = transformers.DripDripTransformerInitializer - transformerInitializerMap["dripFileIlk"] = transformers.DripFileIlkTransformerInitializer - transformerInitializerMap["dripFileRepo"] = transformers.DripFileRepoTransformerInitializer - transformerInitializerMap["dripFileVow"] = transformers.DripFileVowTransfromerInitializer - transformerInitializerMap["flipKick"] = transformers.FlipKickTransformerInitializer - transformerInitializerMap["flopKick"] = transformers.FlopKickTransformerInitializer - transformerInitializerMap["frob"] = transformers.FrobTransformerInitializer - transformerInitializerMap["pitFileDebtCeiling"] = transformers.PitFileDebtCeilingTransformerInitializer - transformerInitializerMap["pitFileIlk"] = transformers.PitFileIlkTransformerInitializer - transformerInitializerMap["pitFileStabilityFee"] = transformers.PitFileStabilityFeeTransformerInitializer - transformerInitializerMap["priceFeed"] = transformers.PriceFeedTransformerInitializer - transformerInitializerMap["tend"] = transformers.TendTransformerInitializer - transformerInitializerMap["vatGrab"] = transformers.VatGrabTransformerInitializer - transformerInitializerMap["vatInit"] = transformers.VatInitTransformerInitializer - transformerInitializerMap["vatMove"] = transformers.VatMoveTransformerInitializer - transformerInitializerMap["vatHeal"] = transformers.VatHealTransformerInitializer - transformerInitializerMap["vatFold"] = transformers.VatFoldTransformerInitializer - transformerInitializerMap["vatSlip"] = transformers.VatSlipTransformerInitializer - transformerInitializerMap["vatToll"] = transformers.VatTollTransformerInitializer - transformerInitializerMap["vatTune"] = transformers.VatTuneTransformerInitializer - transformerInitializerMap["vatFlux"] = transformers.VatFluxTransformerInitializer + transformerInitializerMap[shared2.BiteLabel] = transformers.BiteTransformerInitializer + transformerInitializerMap[shared2.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer + transformerInitializerMap[shared2.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer + transformerInitializerMap[shared2.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer + transformerInitializerMap[shared2.DealLabel] = transformers.DealTransformerInitializer + transformerInitializerMap[shared2.DentLabel] = transformers.DentTransformerInitializer + transformerInitializerMap[shared2.DripDripLabel] = transformers.DripDripTransformerInitializer + transformerInitializerMap[shared2.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer + transformerInitializerMap[shared2.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer + transformerInitializerMap[shared2.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer + transformerInitializerMap[shared2.FlipKickLabel] = transformers.FlipKickTransformerInitializer + transformerInitializerMap[shared2.FlopKickLabel] = transformers.FlopKickTransformerInitializer + transformerInitializerMap[shared2.FrobLabel] = transformers.FrobTransformerInitializer + transformerInitializerMap[shared2.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer + transformerInitializerMap[shared2.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer + transformerInitializerMap[shared2.PitFileStabilityFeeLabel] = transformers.PitFileStabilityFeeTransformerInitializer + transformerInitializerMap[shared2.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer + transformerInitializerMap[shared2.TendLabel] = transformers.TendTransformerInitializer + transformerInitializerMap[shared2.VatGrabLabel] = transformers.VatGrabTransformerInitializer + transformerInitializerMap[shared2.VatInitLabel] = transformers.VatInitTransformerInitializer + transformerInitializerMap[shared2.VatMoveLabel] = transformers.VatMoveTransformerInitializer + transformerInitializerMap[shared2.VatHealLabel] = transformers.VatHealTransformerInitializer + transformerInitializerMap[shared2.VatFoldLabel] = transformers.VatFoldTransformerInitializer + transformerInitializerMap[shared2.VatSlipLabel] = transformers.VatSlipTransformerInitializer + transformerInitializerMap[shared2.VatTollLabel] = transformers.VatTollTransformerInitializer + transformerInitializerMap[shared2.VatTuneLabel] = transformers.VatTuneTransformerInitializer + transformerInitializerMap[shared2.VatFluxLabel] = transformers.VatFluxTransformerInitializer return transformerInitializerMap } diff --git a/pkg/transformers/drip_file/ilk/config.go b/pkg/transformers/drip_file/ilk/config.go index d4bafbf6..de6ad5ca 100644 --- a/pkg/transformers/drip_file/ilk/config.go +++ b/pkg/transformers/drip_file/ilk/config.go @@ -17,7 +17,7 @@ package ilk import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var DripFileIlkConfig = shared.SingleTransformerConfig{ - TransformerName: "drip file ilk", + TransformerName: shared.DripFileIlkLabel, ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, Topic: shared.DripFileIlkSignature, diff --git a/pkg/transformers/drip_file/repo/config.go b/pkg/transformers/drip_file/repo/config.go index 676a8341..c0609644 100644 --- a/pkg/transformers/drip_file/repo/config.go +++ b/pkg/transformers/drip_file/repo/config.go @@ -17,7 +17,7 @@ package repo import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var DripFileRepoConfig = shared.SingleTransformerConfig{ - TransformerName: "drip file repo", + TransformerName: shared.DripFileRepoLabel, ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, Topic: shared.DripFileRepoSignature, diff --git a/pkg/transformers/drip_file/vow/config.go b/pkg/transformers/drip_file/vow/config.go index aeda864f..1bd5e940 100644 --- a/pkg/transformers/drip_file/vow/config.go +++ b/pkg/transformers/drip_file/vow/config.go @@ -17,7 +17,7 @@ package vow import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var DripFileVowConfig = shared.SingleTransformerConfig{ - TransformerName: "drip file vow", + TransformerName: shared.DripFileVowLabel, ContractAddresses: []string{shared.DripContractAddress}, ContractAbi: shared.DripABI, Topic: shared.DripFileVowSignature, diff --git a/pkg/transformers/pit_file/debt_ceiling/config.go b/pkg/transformers/pit_file/debt_ceiling/config.go index 93c483f3..52dae6fc 100644 --- a/pkg/transformers/pit_file/debt_ceiling/config.go +++ b/pkg/transformers/pit_file/debt_ceiling/config.go @@ -19,7 +19,7 @@ import ( ) var DebtCeilingFileConfig = shared.SingleTransformerConfig{ - TransformerName: "pit file debt ceiling", + TransformerName: shared.PitFileDebtCeilingLabel, ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, Topic: shared.PitFileDebtCeilingSignature, diff --git a/pkg/transformers/pit_file/ilk/config.go b/pkg/transformers/pit_file/ilk/config.go index c2def4e2..20d35bc3 100644 --- a/pkg/transformers/pit_file/ilk/config.go +++ b/pkg/transformers/pit_file/ilk/config.go @@ -19,7 +19,7 @@ import ( ) var IlkFileConfig = shared.SingleTransformerConfig{ - TransformerName: "pit file ilk", + TransformerName: shared.PitFileIlkLabel, ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, Topic: shared.PitFileIlkSignature, diff --git a/pkg/transformers/pit_file/stability_fee/config.go b/pkg/transformers/pit_file/stability_fee/config.go index d1894d63..01848543 100644 --- a/pkg/transformers/pit_file/stability_fee/config.go +++ b/pkg/transformers/pit_file/stability_fee/config.go @@ -19,7 +19,7 @@ import ( ) var StabilityFeeFileConfig = shared.SingleTransformerConfig{ - TransformerName: "pit file stability fee", + TransformerName: shared.PitFileStabilityFeeLabel, ContractAddresses: []string{shared.PitContractAddress}, ContractAbi: shared.PitABI, Topic: shared.PitFileStabilityFeeSignature, diff --git a/pkg/transformers/shared/constants.go b/pkg/transformers/shared/constants.go index 6b3c2180..463deee1 100644 --- a/pkg/transformers/shared/constants.go +++ b/pkg/transformers/shared/constants.go @@ -102,4 +102,32 @@ var ( VatTollSignature = GetLogNoteSignature(vatTollMethod) VatTuneSignature = GetLogNoteSignature(vatTuneMethod) VatFluxSignature = GetLogNoteSignature(vatFluxMethod) + + BiteLabel = "bite" + DealLabel = "deal" + CatFileChopLumpLabel = "catFileChopLump" + CatFileFlipLabel = "catFileFlip" + CatFilePitVowLabel = "catFilePitVow" + DentLabel = "dent" + DripDripLabel = "dripDrip" + DripFileIlkLabel = "dripFileIlk" + DripFileRepoLabel = "dripFileRepo" + DripFileVowLabel = "dripFileVow" + FlipKickLabel = "flipKick" + FlopKickLabel = "flopKick" + FrobLabel = "frob" + PitFileDebtCeilingLabel = "pitFileDebtCeiling" + PitFileIlkLabel = "pitFileIlk" + PitFileStabilityFeeLabel = "pitFileStabilityFee" + PriceFeedLabel = "priceFeed" + TendLabel = "tend" + VatHealLabel = "vatHeal" + VatGrabLabel = "vatGrab" + VatInitLabel = "vatInit" + VatMoveLabel = "vatMove" + VatFoldLabel = "vatFold" + VatSlipLabel = "vatSlip" + VatTollLabel = "vatToll" + VatTuneLabel = "vatTune" + VatFluxLabel = "vatFlux" ) diff --git a/pkg/transformers/tend/config.go b/pkg/transformers/tend/config.go index 1bd07c86..d4911e45 100644 --- a/pkg/transformers/tend/config.go +++ b/pkg/transformers/tend/config.go @@ -17,7 +17,7 @@ package tend import "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" var TendConfig = shared.SingleTransformerConfig{ - TransformerName: "tend", + TransformerName: shared.TendLabel, ContractAddresses: []string{shared.FlipperContractAddress}, ContractAbi: shared.FlipperABI, Topic: shared.TendFunctionSignature, diff --git a/pkg/transformers/vat_init/config.go b/pkg/transformers/vat_init/config.go index ed5d34d5..d54abb56 100644 --- a/pkg/transformers/vat_init/config.go +++ b/pkg/transformers/vat_init/config.go @@ -19,7 +19,7 @@ import ( ) var VatInitConfig = shared.SingleTransformerConfig{ - TransformerName: "vat init", + TransformerName: shared.VatInitLabel, ContractAddresses: []string{shared.VatContractAddress}, ContractAbi: shared.VatABI, Topic: shared.VatInitSignature, diff --git a/pkg/transformers/vat_move/config.go b/pkg/transformers/vat_move/config.go index f9da1184..93f25e55 100644 --- a/pkg/transformers/vat_move/config.go +++ b/pkg/transformers/vat_move/config.go @@ -19,7 +19,7 @@ import ( ) var VatMoveConfig = shared.SingleTransformerConfig{ - TransformerName: "vat move", + TransformerName: shared.VatMoveLabel, ContractAddresses: []string{shared.VatContractAddress}, ContractAbi: shared.VatABI, Topic: shared.VatMoveSignature,