From f7ecab9dc25ca9e5ec7f2785c09a3d41a269ff76 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 28 Nov 2018 15:39:46 +0100 Subject: [PATCH 01/33] Fix typo --- pkg/transformers/factories/log_note_transformer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index b237cd87..d489f948 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -41,7 +41,7 @@ func (transformer LogNoteTransformer) Execute() error { transformerName := transformer.Config.TransformerName missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) if err != nil { - log.Printf("Error fetching mising headers in %v transformer: %v", transformerName, err) + log.Printf("Error fetching missing headers in %v transformer: %v", transformerName, err) return err } From 423fdf01b54a1729626ee070c7ec7de05780690b Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 28 Nov 2018 15:40:15 +0100 Subject: [PATCH 02/33] Add log_chunker --- pkg/transformers/shared/log_chunker.go | 57 ++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pkg/transformers/shared/log_chunker.go diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go new file mode 100644 index 00000000..259a7f84 --- /dev/null +++ b/pkg/transformers/shared/log_chunker.go @@ -0,0 +1,57 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +type LogChunker struct { + addressToNames map[string][]string + nameToTopic0 map[string]common.Hash +} + +// Initialises a chunker by creating efficient lookup maps +func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker { + addressToNames := map[string][]string{} + nameToTopic0 := map[string]common.Hash{} + + for _, config := range transformerConfigs { + for _, address := range config.ContractAddresses { + addressToNames[address] = append(addressToNames[address], config.TransformerName) + nameToTopic0[config.TransformerName] = common.HexToHash(config.Topic) + } + } + + return LogChunker{ + addressToNames, + nameToTopic0, + } +} + +// Goes through an array of logs, associating relevant logs with transformers +func (chunker LogChunker) ChunkLogs(logs []types.Log) (chunks map[string][]types.Log) { + for _, log := range logs { + // Topic0 is not unique to each transformer, also need to consider the contract address + relevantTransformers := chunker.addressToNames[log.Address.String()] + for _, transformer := range relevantTransformers { + if chunker.nameToTopic0[transformer] == log.Topics[0] { + chunks[transformer] = append(chunks[transformer], log) + } + } + } + return +} From 45a087f5d1ed3e74ac188e87ec517274f4addfef Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 28 Nov 2018 17:00:04 +0100 Subject: [PATCH 03/33] WIP: First steps in refactoring to aggregate fetching --- libraries/shared/watcher.go | 12 ++++++-- .../factories/log_note_transformer.go | 30 ++++--------------- pkg/transformers/factories/transformer.go | 25 ++++------------ pkg/transformers/shared/transformer.go | 7 +++-- pkg/transformers/transformers.go | 28 ----------------- 5 files changed, 24 insertions(+), 78 deletions(-) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 1e3c47e9..cd566ddf 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -1,6 +1,7 @@ package shared import ( + "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -14,15 +15,22 @@ type Watcher struct { func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { for _, transformerInitializer := range us { - transformer := transformerInitializer(&watcher.DB, watcher.Blockchain) + transformer := transformerInitializer(&watcher.DB) watcher.Transformers = append(watcher.Transformers, transformer) } } func (watcher *Watcher) Execute() error { + // TODO Solve checkedHeadersColumn issue + // TODO Handle start and end numbers in transformers? + var missingHeaders []core.Header + + // TODO Get contract addresses and topic0s + var logs []types.Log + var err error for _, transformer := range watcher.Transformers { - err = transformer.Execute() + err = transformer.Execute(logs, missingHeaders) } return err } diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index d489f948..453d72b0 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -15,10 +15,9 @@ package factories import ( + "github.com/ethereum/go-ethereum/core/types" log "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -28,38 +27,19 @@ type LogNoteTransformer struct { Config shared.TransformerConfig Converter LogNoteConverter Repository Repository - Fetcher shared.SettableLogFetcher } -func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { +func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) shared.Transformer { transformer.Repository.SetDB(db) - transformer.Fetcher.SetBC(bc) return transformer } -func (transformer LogNoteTransformer) Execute() error { +func (transformer LogNoteTransformer) Execute(logs []types.Log, missingHeaders []core.Header) error { transformerName := transformer.Config.TransformerName - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) - if err != nil { - log.Printf("Error fetching missing headers in %v transformer: %v", transformerName, err) - return err - } - // Grab event signature from transformer config - // (Double-array structure required for go-ethereum FilterQuery) - var topic = [][]common.Hash{{common.HexToHash(transformer.Config.Topic)}} - - log.Printf("Fetching %v event logs for %d headers", transformerName, len(missingHeaders)) for _, header := range missingHeaders { - // Fetch the missing logs for a given header - matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header) - if err != nil { - log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) - return err - } - // No matching logs, mark the header as checked for this type of logs - if len(matchingLogs) < 1 { + if len(logs) < 1 { err := transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { log.Printf("Error marking header as checked in %v: %v", transformerName, err) @@ -69,7 +49,7 @@ func (transformer LogNoteTransformer) Execute() error { continue } - models, err := transformer.Converter.ToModels(matchingLogs) + models, err := transformer.Converter.ToModels(logs) if err != nil { log.Printf("Error converting logs in %v: %v", transformerName, err) return err diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 1fe13236..6c1a8a4f 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -15,10 +15,9 @@ package factories import ( + "github.com/ethereum/go-ethereum/core/types" log "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -28,34 +27,20 @@ type Transformer struct { Config shared.TransformerConfig Converter Converter Repository Repository - Fetcher shared.SettableLogFetcher } -func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { +func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transformer { transformer.Repository.SetDB(db) - transformer.Fetcher.SetBC(bc) return transformer } -func (transformer Transformer) Execute() error { +func (transformer Transformer) Execute(logs []types.Log, missingHeaders []core.Header) error { transformerName := transformer.Config.TransformerName config := transformer.Config - topics := [][]common.Hash{{common.HexToHash(config.Topic)}} - missingHeaders, err := transformer.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber) - if err != nil { - log.Printf("Error fetching missing headers in %v transformer: %v \n", transformerName, err) - return err - } - log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders)) - for _, header := range missingHeaders { - logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header) - if err != nil { - log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) - return err - } + for _, header := range missingHeaders { if len(logs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) + err := transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 267077ea..0e5a3f9e 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -16,16 +16,17 @@ package shared import ( "github.com/ethereum/go-ethereum/common" - + "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) type Transformer interface { - Execute() error + Execute(logs []types.Log, missingHeaders []core.Header) error } -type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer +type TransformerInitializer func(db *postgres.DB) Transformer type TransformerConfig struct { TransformerName string diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index f79ac036..73e82d6f 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -52,196 +52,168 @@ var ( Config: bite.BiteConfig, Converter: &bite.BiteConverter{}, Repository: &bite.BiteRepository{}, - Fetcher: &shared.Fetcher{}, }.NewTransformer CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{ Config: chop_lump.CatFileChopLumpConfig, Converter: &chop_lump.CatFileChopLumpConverter{}, Repository: &chop_lump.CatFileChopLumpRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer CatFileFlipTransformerInitializer = factories.LogNoteTransformer{ Config: flip.CatFileFlipConfig, Converter: &flip.CatFileFlipConverter{}, Repository: &flip.CatFileFlipRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{ Config: pit_vow.CatFilePitVowConfig, Converter: &pit_vow.CatFilePitVowConverter{}, Repository: &pit_vow.CatFilePitVowRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer DealTransformerInitializer = factories.LogNoteTransformer{ Config: deal.DealConfig, Converter: &deal.DealConverter{}, Repository: &deal.DealRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer DentTransformerInitializer = factories.LogNoteTransformer{ Config: dent.DentConfig, Converter: &dent.DentConverter{}, Repository: &dent.DentRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer DripDripTransformerInitializer = factories.LogNoteTransformer{ Config: drip_drip.DripDripConfig, Converter: &drip_drip.DripDripConverter{}, Repository: &drip_drip.DripDripRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer DripFileIlkTransformerInitializer = factories.LogNoteTransformer{ Config: ilk2.DripFileIlkConfig, Converter: &ilk2.DripFileIlkConverter{}, Repository: &ilk2.DripFileIlkRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer DripFileRepoTransformerInitializer = factories.LogNoteTransformer{ Config: repo.DripFileRepoConfig, Converter: &repo.DripFileRepoConverter{}, Repository: &repo.DripFileRepoRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer DripFileVowTransfromerInitializer = factories.LogNoteTransformer{ Config: vow.DripFileVowConfig, Converter: &vow.DripFileVowConverter{}, Repository: &vow.DripFileVowRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer FlapKickTransformerInitializer = factories.Transformer{ Config: flap_kick.FlapKickConfig, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, - Fetcher: &shared.Fetcher{}, }.NewTransformer FlipKickTransformerInitializer = factories.Transformer{ Config: flip_kick.FlipKickConfig, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, - Fetcher: &shared.Fetcher{}, }.NewTransformer FlogTransformerInitializer = factories.LogNoteTransformer{ Config: vow_flog.VowFlogConfig, Converter: &vow_flog.VowFlogConverter{}, Repository: &vow_flog.VowFlogRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer FrobTransformerInitializer = factories.Transformer{ Config: frob.FrobConfig, Converter: &frob.FrobConverter{}, Repository: &frob.FrobRepository{}, - Fetcher: &shared.Fetcher{}, }.NewTransformer FlopKickTransformerInitializer = factories.Transformer{ Config: flop_kick.Config, Converter: &flop_kick.FlopKickConverter{}, Repository: &flop_kick.FlopKickRepository{}, - Fetcher: &shared.Fetcher{}, }.NewTransformer PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{ Config: debt_ceiling.DebtCeilingFileConfig, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer PitFileIlkTransformerInitializer = factories.LogNoteTransformer{ Config: ilk.IlkFileConfig, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer PriceFeedTransformerInitializer = factories.LogNoteTransformer{ Config: price_feeds.PriceFeedConfig, Converter: &price_feeds.PriceFeedConverter{}, Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer TendTransformerInitializer = factories.LogNoteTransformer{ Config: tend.TendConfig, Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatInitTransformerInitializer = factories.LogNoteTransformer{ Config: vat_init.VatInitConfig, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatGrabTransformerInitializer = factories.LogNoteTransformer{ Config: vat_grab.VatGrabConfig, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatFoldTransformerInitializer = factories.LogNoteTransformer{ Config: vat_fold.VatFoldConfig, Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatHealTransformerInitializer = factories.LogNoteTransformer{ Config: vat_heal.VatHealConfig, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatMoveTransformerInitializer = factories.LogNoteTransformer{ Config: vat_move.VatMoveConfig, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatSlipTransformerInitializer = factories.LogNoteTransformer{ Config: vat_slip.VatSlipConfig, Converter: &vat_slip.VatSlipConverter{}, Repository: &vat_slip.VatSlipRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatTollTransformerInitializer = factories.LogNoteTransformer{ Config: vat_toll.VatTollConfig, Converter: &vat_toll.VatTollConverter{}, Repository: &vat_toll.VatTollRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatTuneTransformerInitializer = factories.LogNoteTransformer{ Config: vat_tune.VatTuneConfig, Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer VatFluxTransformerInitializer = factories.LogNoteTransformer{ Config: vat_flux.VatFluxConfig, Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, - Fetcher: &shared.Fetcher{}, }.NewLogNoteTransformer ) From b5dab1e83b126a00524766ba5bcef1ebf8c1cc6b Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 29 Nov 2018 11:48:16 +0100 Subject: [PATCH 04/33] Nuke erc20 transformer --- cmd/erc20.go | 71 ------ examples/constants/constants.go | 21 -- examples/erc20_test_helpers/database.go | 27 --- examples/erc20_test_helpers/mocks/mocks.go | 106 --------- examples/erc20_watcher/ERC20WatcherREADME.md | 19 -- examples/erc20_watcher/config.go | 33 --- .../every_block/every_block_suite_test.go | 33 --- examples/erc20_watcher/every_block/fetcher.go | 71 ------ .../erc20_watcher/every_block/fetcher_test.go | 82 ------- .../every_block/integration_test.go | 83 ------- examples/erc20_watcher/every_block/model.go | 21 -- .../erc20_watcher/every_block/repository.go | 91 -------- .../every_block/repository_test.go | 207 ------------------ .../erc20_watcher/every_block/transformer.go | 118 ---------- .../every_block/transformer_test.go | 188 ---------------- .../erc20_watcher/every_block/transformers.go | 28 --- 16 files changed, 1199 deletions(-) delete mode 100644 cmd/erc20.go delete mode 100644 examples/constants/constants.go delete mode 100644 examples/erc20_test_helpers/database.go delete mode 100644 examples/erc20_test_helpers/mocks/mocks.go delete mode 100644 examples/erc20_watcher/ERC20WatcherREADME.md delete mode 100644 examples/erc20_watcher/config.go delete mode 100644 examples/erc20_watcher/every_block/every_block_suite_test.go delete mode 100644 examples/erc20_watcher/every_block/fetcher.go delete mode 100644 examples/erc20_watcher/every_block/fetcher_test.go delete mode 100644 examples/erc20_watcher/every_block/integration_test.go delete mode 100644 examples/erc20_watcher/every_block/model.go delete mode 100644 examples/erc20_watcher/every_block/repository.go delete mode 100644 examples/erc20_watcher/every_block/repository_test.go delete mode 100644 examples/erc20_watcher/every_block/transformer.go delete mode 100644 examples/erc20_watcher/every_block/transformer_test.go delete mode 100644 examples/erc20_watcher/every_block/transformers.go diff --git a/cmd/erc20.go b/cmd/erc20.go deleted file mode 100644 index d0d8f112..00000000 --- a/cmd/erc20.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "time" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/libraries/shared" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" -) - -// erc20Cmd represents the erc20 command -var erc20Cmd = &cobra.Command{ - Use: "erc20", - Short: "Fetches and persists token supply", - Long: `Fetches the totalSupply for the configured token from each block and persists it in Vulcanize DB. -vulcanizedb erc20 --config environments/public - -Expects an ethereum node to be running and requires a .toml config file: - - [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - - [client] - ipcPath = "/Users/user/Library/Ethereum/geth.ipc" -`, - Run: func(cmd *cobra.Command, args []string) { - watchERC20s() - }, -} - -func watchERC20s() { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - blockChain := getBlockChain() - db, err := postgres.NewDB(databaseConfig, blockChain.Node()) - if err != nil { - log.Fatal("Failed to initialize database.") - } - watcher := shared.Watcher{ - DB: *db, - Blockchain: blockChain, - } - - watcher.AddTransformers(every_block.TransformerInitializers()) - for range ticker.C { - watcher.Execute() - } -} - -func init() { - rootCmd.AddCommand(erc20Cmd) -} diff --git a/examples/constants/constants.go b/examples/constants/constants.go deleted file mode 100644 index abc84f94..00000000 --- a/examples/constants/constants.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package constants - -//Contract Addresses -var DaiContractAddress = "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" - -//Abis -var DaiAbiString = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"stop","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"name_","type":"bytes32"}],"name":"setName","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"src","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"stopped","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"push","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"move","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"start","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"src","type":"address"},{"name":"guy","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"wad","type":"uint256"}],"name":"pull","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"symbol_","type":"bytes32"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Transfer","type":"event"}]` diff --git a/examples/erc20_test_helpers/database.go b/examples/erc20_test_helpers/database.go deleted file mode 100644 index 15946db1..00000000 --- a/examples/erc20_test_helpers/database.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package erc20_test_helpers - -type TokenSupplyDBRow struct { - ID int64 - Supply int64 - BlockID int64 `db:"block_id"` - TokenAddress string `db:"token_address"` -} - -type TransferDBRow struct { - ID int64 `db:"id"` - VulcanizeLogID int64 `db:"vulcanize_log_id"` -} diff --git a/examples/erc20_test_helpers/mocks/mocks.go b/examples/erc20_test_helpers/mocks/mocks.go deleted file mode 100644 index b2944170..00000000 --- a/examples/erc20_test_helpers/mocks/mocks.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mocks - -import ( - "math/big" - - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/fakes" -) - -type Fetcher struct { - ContractAddress string - Abi string - FetchedBlocks []int64 - BlockChain core.BlockChain - supply big.Int -} - -func (f *Fetcher) SetSupply(supply string) { - f.supply.SetString(supply, 10) -} - -func (f Fetcher) GetBlockChain() core.BlockChain { - return f.BlockChain -} - -func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) { - f.Abi = contractAbi - f.ContractAddress = contractAddress - f.FetchedBlocks = append(f.FetchedBlocks, blockNumber) - - accumulator := big.NewInt(1) - f.supply.Add(&f.supply, accumulator) - - return f.supply, nil -} - -type TotalSupplyRepository struct { - TotalSuppliesCreated []every_block.TokenSupply - MissingBlockNumbers []int64 - StartingBlock int64 - EndingBlock int64 -} - -func (fr *TotalSupplyRepository) Create(supply every_block.TokenSupply) error { - fr.TotalSuppliesCreated = append(fr.TotalSuppliesCreated, supply) - return nil -} - -func (fr *TotalSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { - fr.StartingBlock = startingBlock - fr.EndingBlock = highestBlock - return fr.MissingBlockNumbers, nil -} - -func (fr *TotalSupplyRepository) SetMissingBlocks(missingBlocks []int64) { - fr.MissingBlockNumbers = missingBlocks -} - -type FailureRepository struct { - createFail bool - missingBlocksFail bool - missingBlocksNumbers []int64 -} - -func (fr *FailureRepository) Create(supply every_block.TokenSupply) error { - if fr.createFail { - return fakes.FakeError - } else { - return nil - } -} - -func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { - if fr.missingBlocksFail { - return []int64{}, fakes.FakeError - } else { - return fr.missingBlocksNumbers, nil - } -} - -func (fr *FailureRepository) SetCreateFail(fail bool) { - fr.createFail = fail -} - -func (fr *FailureRepository) SetMissingBlocksFail(fail bool) { - fr.missingBlocksFail = fail -} - -func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) { - fr.missingBlocksNumbers = missingBlocks -} diff --git a/examples/erc20_watcher/ERC20WatcherREADME.md b/examples/erc20_watcher/ERC20WatcherREADME.md deleted file mode 100644 index 97ecdfe4..00000000 --- a/examples/erc20_watcher/ERC20WatcherREADME.md +++ /dev/null @@ -1,19 +0,0 @@ -# ERC20 Transformers - -## Description -The Transformers in this directory are associated with contract functions and events that conform to the [ERC20 Token interface](https://theethereum.wiki/w/index.php/ERC20_Token_Standard#The_ERC20_Token_Standard_Interface). - -See `libraries/shared/TransformerREADME.md` for further information regarding the Transformer interface. - -## Configuration -In addition to environment configuration mentioned in the main VulcanizeDB README, the ERC20 transformers also need to be configured with contract information for the desired token(s) to be watched. This configuration file is located at `./vulcanizedb/examples/erc20_watcher/config.go`. - -## ERC20 Functions -The `everyblock` directory contains transformers that fetch data from the contract itself, via one of the standard functions. - -Currently, the `totalSupply` function transformer has been implemented. This transformer will fetch the total supply for the given contract address and persist `total_supply` records in the database. - -## Running the transformers -1. If running against a local node, make sure that the node has been started. -1. In a separate terminal run the following command: -`./vulcanizedb erc20 --config ` diff --git a/examples/erc20_watcher/config.go b/examples/erc20_watcher/config.go deleted file mode 100644 index 880d2143..00000000 --- a/examples/erc20_watcher/config.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package erc20_watcher - -import "github.com/vulcanize/vulcanizedb/examples/constants" - -type ContractConfig struct { - Address string - Abi string - FirstBlock int64 - LastBlock int64 - Name string -} - -var DaiConfig = ContractConfig{ - Address: constants.DaiContractAddress, - Abi: constants.DaiAbiString, - FirstBlock: int64(4752008), - LastBlock: -1, - Name: "Dai", -} diff --git a/examples/erc20_watcher/every_block/every_block_suite_test.go b/examples/erc20_watcher/every_block/every_block_suite_test.go deleted file mode 100644 index 69d12677..00000000 --- a/examples/erc20_watcher/every_block/every_block_suite_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - "io/ioutil" -) - -func TestEveryBlock(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "EveryBlock Suite") -} - -var _ = BeforeSuite(func() { - log.SetOutput(ioutil.Discard) -}) diff --git a/examples/erc20_watcher/every_block/fetcher.go b/examples/erc20_watcher/every_block/fetcher.go deleted file mode 100644 index 07681fe4..00000000 --- a/examples/erc20_watcher/every_block/fetcher.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "fmt" - log "github.com/sirupsen/logrus" - "math/big" - - "github.com/vulcanize/vulcanizedb/pkg/core" -) - -type ERC20FetcherInterface interface { - FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) - GetBlockChain() core.BlockChain -} - -func NewFetcher(blockchain core.BlockChain) Fetcher { - return Fetcher{ - Blockchain: blockchain, - } -} - -type Fetcher struct { - Blockchain core.BlockChain - ContractAbi string - ContractAddress string -} - -type fetcherError struct { - err string - fetchMethod string -} - -func (fe *fetcherError) Error() string { - return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err) -} - -func newFetcherError(err error, fetchMethod string) *fetcherError { - e := fetcherError{err.Error(), fetchMethod} - log.Info(e.Error()) - return &e -} - -func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) { - method := "totalSupply" - var result = new(big.Int) - err := f.Blockchain.FetchContractData(contractAbi, contractAddress, method, nil, &result, blockNumber) - - if err != nil { - return *result, newFetcherError(err, method) - } - - return *result, nil -} - -func (f Fetcher) GetBlockChain() core.BlockChain { - return f.Blockchain -} diff --git a/examples/erc20_watcher/every_block/fetcher_test.go b/examples/erc20_watcher/every_block/fetcher_test.go deleted file mode 100644 index 9a51fcc1..00000000 --- a/examples/erc20_watcher/every_block/fetcher_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/big" - - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/geth/client" - rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" - "github.com/vulcanize/vulcanizedb/pkg/geth/node" -) - -var _ = Describe("ERC20 Fetcher", func() { - blockNumber := int64(5502914) - - Describe("FetchSupplyOf", func() { - It("fetches data from the blockchain with the correct arguments", func() { - fakeBlockchain := fakes.NewMockBlockChain() - testFetcher := every_block.NewFetcher(fakeBlockchain) - testAbi := "testAbi" - testContractAddress := "testContractAddress" - _, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber) - - Expect(err).NotTo(HaveOccurred()) - expectedResult := big.Int{} - expected := &expectedResult - fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber) - }) - - It("fetches a token's total supply at the given block height", func() { - infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" - rawRpcClient, err := rpc.Dial(infuraIPC) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) - ethClient := ethclient.NewClient(rawRpcClient) - blockChainClient := client.NewEthClient(ethClient) - node := node.MakeNode(rpcClient) - transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) - realFetcher := every_block.NewFetcher(blockChain) - result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber) - - Expect(err).NotTo(HaveOccurred()) - expectedResult := big.Int{} - expectedResult.SetString("27647235749155415536952630", 10) - Expect(result).To(Equal(expectedResult)) - }) - - It("returns an error if the call to the blockchain fails", func() { - blockChain := fakes.NewMockBlockChain() - blockChain.SetFetchContractDataErr(fakes.FakeError) - errorFetcher := every_block.NewFetcher(blockChain) - result, err := errorFetcher.FetchSupplyOf("", "", 0) - - Expect(result.String()).To(Equal("0")) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("totalSupply")) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - }) - }) -}) diff --git a/examples/erc20_watcher/every_block/integration_test.go b/examples/erc20_watcher/every_block/integration_test.go deleted file mode 100644 index ed13725a..00000000 --- a/examples/erc20_watcher/every_block/integration_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/big" - "strconv" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/test_config" -) - -func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) { - blockNumberString := strconv.FormatInt(blockNumber, 10) - lastBlockOnChain := big.Int{} - lastBlockOnChain.SetString(blockNumberString, 10) - blockChain.SetLastBlock(&lastBlockOnChain) -} - -var _ = Describe("Everyblock transformers", func() { - var db *postgres.DB - var blockChain *fakes.MockBlockChain - var blockNumber int64 - var blockId int64 - var err error - - BeforeEach(func() { - blockChain = fakes.NewMockBlockChain() - blockNumber = erc20_watcher.DaiConfig.FirstBlock - lastBlockNumber := blockNumber + 1 - node := test_config.NewTestNode() - db = test_config.NewTestDB(node) - test_config.CleanTestDB(db) - setLastBlockOnChain(blockChain, lastBlockNumber) - - blockRepository := repositories.NewBlockRepository(db) - - blockId, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) - Expect(err).NotTo(HaveOccurred()) - _, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: lastBlockNumber}) - Expect(err).NotTo(HaveOccurred()) - }) - - It("creates a token_supply record for each block in the given range", func() { - initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig} - transformer := initializer.NewTokenSupplyTransformer(db, blockChain) - transformer.Execute() - - var tokenSupplyCount int - err := db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount) - Expect(err).ToNot(HaveOccurred()) - Expect(tokenSupplyCount).To(Equal(1)) - - var tokenSupply erc20_test_helpers.TokenSupplyDBRow - err = db.Get(&tokenSupply, `SELECT * from token_supply where block_id = $1`, blockId) - Expect(err).ToNot(HaveOccurred()) - Expect(tokenSupply.BlockID).To(Equal(blockId)) - Expect(tokenSupply.TokenAddress).To(Equal(constants.DaiContractAddress)) - Expect(tokenSupply.Supply).To(Equal(int64(0))) - }) -}) diff --git a/examples/erc20_watcher/every_block/model.go b/examples/erc20_watcher/every_block/model.go deleted file mode 100644 index fc973b9b..00000000 --- a/examples/erc20_watcher/every_block/model.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -type TokenSupply struct { - Value string - TokenAddress string - BlockNumber int64 -} diff --git a/examples/erc20_watcher/every_block/repository.go b/examples/erc20_watcher/every_block/repository.go deleted file mode 100644 index c641665b..00000000 --- a/examples/erc20_watcher/every_block/repository.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "fmt" - log "github.com/sirupsen/logrus" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" -) - -type ERC20RepositoryInterface interface { - Create(supply TokenSupply) error - MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) -} - -type TokenSupplyRepository struct { - *postgres.DB -} - -type repositoryError struct { - err string - msg string - blockNumber int64 -} - -func (re *repositoryError) Error() string { - return fmt.Sprintf(re.msg, re.blockNumber, re.err) -} - -func newRepositoryError(err error, msg string, blockNumber int64) error { - e := repositoryError{err.Error(), msg, blockNumber} - log.Println(e.Error()) - return &e -} - -const ( - GetBlockError = "Error fetching block number %d: %s" - InsertTokenSupplyError = "Error inserting token_supply for block number %d: %s" - MissingBlockError = "Error finding missing token_supply records starting at block %d: %s" -) - -func (tsp *TokenSupplyRepository) Create(supply TokenSupply) error { - var blockId int - err := tsp.DB.Get(&blockId, `SELECT id FROM blocks WHERE number = $1 AND eth_node_id = $2`, supply.BlockNumber, tsp.NodeID) - if err != nil { - return newRepositoryError(err, GetBlockError, supply.BlockNumber) - } - - _, err = tsp.DB.Exec( - `INSERT INTO token_supply (supply, token_address, block_id) - VALUES($1, $2, $3)`, - supply.Value, supply.TokenAddress, blockId) - if err != nil { - return newRepositoryError(err, InsertTokenSupplyError, supply.BlockNumber) - } - return nil -} - -func (tsp *TokenSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { - blockNumbers := make([]int64, 0) - - err := tsp.DB.Select( - &blockNumbers, - `SELECT number FROM BLOCKS - LEFT JOIN token_supply ON blocks.id = block_id - WHERE block_id ISNULL - AND eth_node_id = $1 - AND number >= $2 - AND number <= $3 - LIMIT 20`, - tsp.NodeID, - startingBlock, - highestBlock, - ) - if err != nil { - return []int64{}, newRepositoryError(err, MissingBlockError, startingBlock) - } - return blockNumbers, err -} diff --git a/examples/erc20_watcher/every_block/repository_test.go b/examples/erc20_watcher/every_block/repository_test.go deleted file mode 100644 index a508acd1..00000000 --- a/examples/erc20_watcher/every_block/repository_test.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/rand" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("ERC20 Token Repository", func() { - var db *postgres.DB - var blockId int64 - var blockNumber int64 - var repository every_block.TokenSupplyRepository - var blockRepository repositories.BlockRepository - testAddress := "abc" - - BeforeEach(func() { - node := test_config.NewTestNode() - db = test_config.NewTestDB(node) - test_config.CleanTestDB(db) - repository = every_block.TokenSupplyRepository{DB: db} - blockRepository = *repositories.NewBlockRepository(db) - blockNumber = rand.Int63() - blockId = test_config.NewTestBlock(blockNumber, blockRepository) - }) - - Describe("Create", func() { - It("creates a token supply record", func() { - supply := supplyModel(blockNumber, testAddress, "100") - err := repository.Create(supply) - Expect(err).NotTo(HaveOccurred()) - - dbResult := erc20_test_helpers.TokenSupplyDBRow{} - expectedTokenSupply := erc20_test_helpers.TokenSupplyDBRow{ - Supply: int64(100), - BlockID: blockId, - TokenAddress: testAddress, - } - - var count int - err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(1)) - - err = repository.DB.QueryRowx(`SELECT * FROM token_supply`).StructScan(&dbResult) - Expect(err).NotTo(HaveOccurred()) - Expect(dbResult.Supply).To(Equal(expectedTokenSupply.Supply)) - Expect(dbResult.BlockID).To(Equal(expectedTokenSupply.BlockID)) - Expect(dbResult.TokenAddress).To(Equal(expectedTokenSupply.TokenAddress)) - }) - - It("returns an error if fetching the block's id from the database fails", func() { - errorSupply := supplyModel(-1, "", "") - err := repository.Create(errorSupply) - - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("sql")) - Expect(err.Error()).To(ContainSubstring("block number -1")) - }) - - It("returns an error if inserting the token_supply fails", func() { - errorSupply := supplyModel(blockNumber, "", "") - err := repository.Create(errorSupply) - - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("pq")) - Expect(err.Error()).To(ContainSubstring("token_supply for block number")) - }) - }) - - Describe("When there are multiple nodes", func() { - var node2DB *postgres.DB - var node2BlockRepo *repositories.BlockRepository - var node2BlockId int64 - var node2TokenSupplyRepo every_block.TokenSupplyRepository - var tokenSupply every_block.TokenSupply - - BeforeEach(func() { - node2DB = createDbForAnotherNode() - - //create another block with the same number on node2 - node2BlockRepo = repositories.NewBlockRepository(node2DB) - node2BlockId = test_config.NewTestBlock(blockNumber, *node2BlockRepo) - - tokenSupply = supplyModel(blockNumber, "abc", "100") - node2TokenSupplyRepo = every_block.TokenSupplyRepository{DB: node2DB} - }) - - It("only creates token_supply records for the current node (node2)", func() { - err := node2TokenSupplyRepo.Create(tokenSupply) - Expect(err).NotTo(HaveOccurred()) - - var tokenSupplies []erc20_test_helpers.TokenSupplyDBRow - err = node2TokenSupplyRepo.DB.Select(&tokenSupplies, `SELECT * FROM token_supply`) - Expect(err).NotTo(HaveOccurred()) - Expect(len(tokenSupplies)).To(Equal(1)) - Expect(tokenSupplies[0].BlockID).To(Equal(node2BlockId)) - }) - - It("only includes missing block numbers for the current node", func() { - //create token_supply on original node - err := repository.Create(tokenSupply) - Expect(err).NotTo(HaveOccurred()) - - originalNodeMissingBlocks, err := repository.MissingBlocks(blockNumber, blockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(originalNodeMissingBlocks)).To(Equal(0)) - - node2MissingBlocks, err := node2TokenSupplyRepo.MissingBlocks(blockNumber, blockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(node2MissingBlocks)).To(Equal(1)) - }) - }) - - Describe("MissingBlocks", func() { - It("returns the block numbers for which an associated TokenSupply record hasn't been created", func() { - createTokenSupplyFor(repository, blockNumber) - - newBlockNumber := blockNumber + 1 - test_config.NewTestBlock(newBlockNumber, blockRepository) - blocks, err := repository.MissingBlocks(blockNumber, newBlockNumber) - - Expect(blocks).To(ConsistOf(newBlockNumber)) - Expect(err).NotTo(HaveOccurred()) - }) - - It("only returns blocks within the given range", func() { - newBlockNumber := blockNumber + 1 - test_config.NewTestBlock(newBlockNumber, blockRepository) - blocks, err := repository.MissingBlocks(blockNumber, blockNumber) - - Expect(blocks).NotTo(ConsistOf(newBlockNumber)) - Expect(err).NotTo(HaveOccurred()) - }) - - It("does not return numbers that already have an associated TokenSupply record", func() { - createTokenSupplyFor(repository, blockNumber) - blocks, err := repository.MissingBlocks(blockNumber, blockNumber) - - Expect(blocks).To(BeEmpty()) - Expect(err).NotTo(HaveOccurred()) - }) - }) - - It("deletes the token supply record when the associated block is deleted", func() { - err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"}) - Expect(err).NotTo(HaveOccurred()) - - var count int - err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(1)) - - _, err = db.Query(`DELETE FROM blocks`) - Expect(err).NotTo(HaveOccurred()) - - err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(0)) - }) -}) - -func supplyModel(blockNumber int64, tokenAddress string, supplyValue string) every_block.TokenSupply { - return every_block.TokenSupply{ - Value: supplyValue, - TokenAddress: tokenAddress, - BlockNumber: int64(blockNumber), - } -} - -func createTokenSupplyFor(repository every_block.TokenSupplyRepository, blockNumber int64) { - err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"}) - Expect(err).NotTo(HaveOccurred()) -} - -func createDbForAnotherNode() *postgres.DB { - anotherNode := core.Node{ - GenesisBlock: "GENESIS", - NetworkID: 1, - ID: "testNodeId", - ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9", - } - - return test_config.NewTestDB(anotherNode) -} diff --git a/examples/erc20_watcher/every_block/transformer.go b/examples/erc20_watcher/every_block/transformer.go deleted file mode 100644 index 81ff4370..00000000 --- a/examples/erc20_watcher/every_block/transformer.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "fmt" - "math/big" - - log "github.com/sirupsen/logrus" - - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type Transformer struct { - Fetcher ERC20FetcherInterface - Repository ERC20RepositoryInterface - Config erc20_watcher.ContractConfig -} - -func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) { - t.Config = config -} - -type TokenSupplyTransformerInitializer struct { - Config erc20_watcher.ContractConfig -} - -func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer { - fetcher := NewFetcher(blockchain) - repository := TokenSupplyRepository{DB: db} - transformer := Transformer{ - Fetcher: &fetcher, - Repository: &repository, - Config: i.Config, - } - - return transformer -} - -const ( - FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s" - FetchingSupplyError = "Error fetching supply for block %d: %s" - CreateSupplyError = "Error inserting token_supply for block %d: %s" -) - -type transformerError struct { - err string - blockNumber int64 - msg string -} - -func (te *transformerError) Error() string { - return fmt.Sprintf(te.msg, te.blockNumber, te.err) -} - -func newTransformerError(err error, blockNumber int64, msg string) error { - e := transformerError{err.Error(), blockNumber, msg} - log.Println(e.Error()) - return &e -} - -func (t Transformer) Execute() error { - var upperBoundBlock int64 - blockchain := t.Fetcher.GetBlockChain() - lastBlock := blockchain.LastBlock().Int64() - - if t.Config.LastBlock == -1 { - upperBoundBlock = lastBlock - } else { - upperBoundBlock = t.Config.LastBlock - } - - blocks, err := t.Repository.MissingBlocks(t.Config.FirstBlock, upperBoundBlock) - - if err != nil { - return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError) - } - - log.Printf("Fetching totalSupply for %d blocks", len(blocks)) - for _, blockNumber := range blocks { - totalSupply, err := t.Fetcher.FetchSupplyOf(t.Config.Abi, t.Config.Address, blockNumber) - - if err != nil { - return newTransformerError(err, blockNumber, FetchingSupplyError) - } - model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber) - err = t.Repository.Create(model) - - if err != nil { - return newTransformerError(err, blockNumber, CreateSupplyError) - } - } - - return nil -} - -func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int64) TokenSupply { - return TokenSupply{ - Value: totalSupply.String(), - TokenAddress: address, - BlockNumber: blockNumber, - } -} diff --git a/examples/erc20_watcher/every_block/transformer_test.go b/examples/erc20_watcher/every_block/transformer_test.go deleted file mode 100644 index 17439478..00000000 --- a/examples/erc20_watcher/every_block/transformer_test.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/big" - "math/rand" - "strconv" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/fakes" -) - -var testContractConfig = erc20_watcher.ContractConfig{ - Address: constants.DaiContractAddress, - Abi: constants.DaiAbiString, - FirstBlock: int64(4752008), - LastBlock: int64(5750050), - Name: "Dai", -} - -var config = testContractConfig - -var _ = Describe("Everyblock transformer", func() { - var fetcher mocks.Fetcher - var repository mocks.TotalSupplyRepository - var transformer every_block.Transformer - var blockChain *fakes.MockBlockChain - var initialSupply = "27647235749155415536952630" - var initialSupplyPlusOne = "27647235749155415536952631" - var initialSupplyPlusTwo = "27647235749155415536952632" - var initialSupplyPlusThree = "27647235749155415536952633" - var defaultLastBlock = big.Int{} - - BeforeEach(func() { - blockChain = fakes.NewMockBlockChain() - blockChain.SetLastBlock(&defaultLastBlock) - fetcher = mocks.Fetcher{BlockChain: blockChain} - fetcher.SetSupply(initialSupply) - repository = mocks.TotalSupplyRepository{} - repository.SetMissingBlocks([]int64{config.FirstBlock}) - //setting the mock repository to return the first block as the missing blocks - - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &repository, - } - transformer.SetConfiguration(config) - }) - - It("fetches and persists the total supply of a token for a single block", func() { - err := transformer.Execute() - Expect(err).NotTo(HaveOccurred()) - - Expect(len(fetcher.FetchedBlocks)).To(Equal(1)) - Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock)) - Expect(fetcher.Abi).To(Equal(config.Abi)) - Expect(fetcher.ContractAddress).To(Equal(config.Address)) - - Expect(repository.StartingBlock).To(Equal(config.FirstBlock)) - Expect(repository.EndingBlock).To(Equal(config.LastBlock)) - Expect(len(repository.TotalSuppliesCreated)).To(Equal(1)) - expectedSupply := big.Int{} - expectedSupply.SetString(initialSupply, 10) - expectedSupply.Add(&expectedSupply, big.NewInt(1)) - - Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(expectedSupply.String())) - }) - - It("retrieves the total supply for every missing block", func() { - missingBlocks := []int64{ - config.FirstBlock, - config.FirstBlock + 1, - config.FirstBlock + 2, - } - repository.SetMissingBlocks(missingBlocks) - transformer.Execute() - - Expect(len(fetcher.FetchedBlocks)).To(Equal(3)) - Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock, config.FirstBlock+1, config.FirstBlock+2)) - Expect(fetcher.Abi).To(Equal(config.Abi)) - Expect(fetcher.ContractAddress).To(Equal(config.Address)) - - Expect(len(repository.TotalSuppliesCreated)).To(Equal(3)) - Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(initialSupplyPlusOne)) - Expect(repository.TotalSuppliesCreated[1].Value).To(Equal(initialSupplyPlusTwo)) - Expect(repository.TotalSuppliesCreated[2].Value).To(Equal(initialSupplyPlusThree)) - }) - - It("uses the set contract configuration", func() { - repository.SetMissingBlocks([]int64{testContractConfig.FirstBlock}) - transformer.SetConfiguration(testContractConfig) - err := transformer.Execute() - Expect(err).NotTo(HaveOccurred()) - - Expect(fetcher.FetchedBlocks).To(ConsistOf(testContractConfig.FirstBlock)) - Expect(fetcher.Abi).To(Equal(testContractConfig.Abi)) - Expect(fetcher.ContractAddress).To(Equal(testContractConfig.Address)) - - Expect(repository.StartingBlock).To(Equal(testContractConfig.FirstBlock)) - Expect(repository.EndingBlock).To(Equal(testContractConfig.LastBlock)) - Expect(len(repository.TotalSuppliesCreated)).To(Equal(1)) - expectedTokenSupply := every_block.TokenSupply{ - Value: initialSupplyPlusOne, - TokenAddress: testContractConfig.Address, - BlockNumber: testContractConfig.FirstBlock, - } - Expect(repository.TotalSuppliesCreated[0]).To(Equal(expectedTokenSupply)) - }) - - It("uses the most recent block if the Config.LastBlock is -1", func() { - testContractConfig.LastBlock = -1 - transformer.SetConfiguration(testContractConfig) - - randomBlockNumber := rand.Int63() - numberToString := strconv.FormatInt(randomBlockNumber, 10) - mostRecentBlock := big.Int{} - mostRecentBlock.SetString(numberToString, 10) - - blockChain.SetLastBlock(&mostRecentBlock) - - err := transformer.Execute() - Expect(err).NotTo(HaveOccurred()) - - Expect(repository.EndingBlock).To(Equal(randomBlockNumber)) - }) - - It("returns an error if the call to get missing blocks fails", func() { - failureRepository := mocks.FailureRepository{} - failureRepository.SetMissingBlocksFail(true) - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &failureRepository, - } - err := transformer.Execute() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - Expect(err.Error()).To(ContainSubstring("fetching missing blocks")) - }) - - It("returns an error if the call to the blockChain fails", func() { - failureBlockchain := fakes.NewMockBlockChain() - failureBlockchain.SetLastBlock(&defaultLastBlock) - failureBlockchain.SetFetchContractDataErr(fakes.FakeError) - fetcher := every_block.NewFetcher(failureBlockchain) - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &repository, - } - err := transformer.Execute() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - Expect(err.Error()).To(ContainSubstring("supply")) - }) - - It("returns an error if the call to save the token_supply fails", func() { - failureRepository := mocks.FailureRepository{} - failureRepository.SetMissingBlocks([]int64{config.FirstBlock}) - failureRepository.SetCreateFail(true) - - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &failureRepository, - } - err := transformer.Execute() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - Expect(err.Error()).To(ContainSubstring("supply")) - }) -}) diff --git a/examples/erc20_watcher/every_block/transformers.go b/examples/erc20_watcher/every_block/transformers.go deleted file mode 100644 index f54b3528..00000000 --- a/examples/erc20_watcher/every_block/transformers.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -func TransformerInitializers() []shared.TransformerInitializer { - config := erc20_watcher.DaiConfig - initializer := TokenSupplyTransformerInitializer{config} - return []shared.TransformerInitializer{ - initializer.NewTokenSupplyTransformer, - } -} From c26736dc9e5d52ceb72e99c2ad1e93b67c7952e4 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 30 Nov 2018 17:28:52 +0100 Subject: [PATCH 05/33] WIP continue on aggregate fetching architecture --- cmd/continuousLogSync.go | 56 ++--- libraries/shared/watcher.go | 54 ++++- pkg/transformers/shared/log_fetcher.go | 28 +-- pkg/transformers/transformers.go | 282 ++++++++++++++----------- 4 files changed, 245 insertions(+), 175 deletions(-) diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 9c23af1b..9f081ea3 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -92,34 +92,34 @@ func getTransformerInititalizers(transformerNames []string) []shared2.Transforme func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { transformerInitializerMap := make(map[string]shared2.TransformerInitializer) - transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer - transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer - transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer - transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer - transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer - transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer - transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer - transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer - transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer - transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer - transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer - transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer - transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer - transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer - transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer - transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer - transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer - transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer - transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer - transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer - transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer - transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer - transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer - transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer - transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer - transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer - transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer - transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer + transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer + transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer + transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer + transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer + transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer + transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer + transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer return transformerInitializerMap } diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index cd566ddf..2820ad1d 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -1,9 +1,10 @@ package shared import ( - "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) @@ -11,6 +12,35 @@ type Watcher struct { Transformers []shared.Transformer DB postgres.DB Blockchain core.BlockChain + Fetcher shared.LogFetcher + Chunker shared.LogChunker + Addresses []common.Address + Topics []common.Hash +} + +func NewWatcher(db postgres.DB, bc core.BlockChain) Watcher { + transformerConfigs := transformers.TransformerConfigs() + var contractAddresses []common.Address + var topic0s []common.Hash + + for _, config := range transformerConfigs { + for _, address := range config.ContractAddresses { + contractAddresses = append(contractAddresses, common.HexToAddress(address)) + } + topic0s = append(topic0s, common.HexToHash(config.Topic)) + } + + chunker := shared.NewLogChunker(transformerConfigs) + fetcher := shared.NewFetcher(bc) + + return Watcher{ + DB: db, + Blockchain: bc, + Fetcher: fetcher, + Chunker: chunker, + Addresses: contractAddresses, + Topics: topic0s, + } } func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { @@ -23,14 +53,24 @@ func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { func (watcher *Watcher) Execute() error { // TODO Solve checkedHeadersColumn issue // TODO Handle start and end numbers in transformers? - var missingHeaders []core.Header + missingHeaders, err := shared.MissingHeaders(0, -1, &watcher.DB, "") - // TODO Get contract addresses and topic0s - var logs []types.Log + for _, header := range missingHeaders { + // TODO Extend FetchLogs for doing several blocks at a time + logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header) + if err != nil { + // TODO Handle fetch error in watcher + return err + } - var err error - for _, transformer := range watcher.Transformers { - err = transformer.Execute(logs, missingHeaders) + chunkedLogs := watcher.Chunker.ChunkLogs(logs) + + for _, transformer := range watcher.Transformers { + // TODO delegate log chunks to respective transformers + // Need to get the transformer name... :/ + logChunk := chunkedLogs["transformerName"] + err = transformer.Execute(logChunk, missingHeaders) + } } return err } diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index 0e6ad7ce..47f6f77d 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -22,8 +22,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) +// TODO Check if Fetcher can be simplified with aggregate logic + type LogFetcher interface { - FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) + FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) } type SettableLogFetcher interface { @@ -45,23 +47,21 @@ func NewFetcher(blockchain core.BlockChain) Fetcher { } } -func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { - addresses := hexStringsToAddresses(contractAddresses) +// Checks all topic0s, on all addresses, fetching matching logs for the given header +func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) { blockHash := common.HexToHash(header.Hash) query := ethereum.FilterQuery{ BlockHash: &blockHash, Addresses: addresses, - Topics: topics, - } - return fetcher.blockChain.GetEthLogsWithCustomQuery(query) -} - -func hexStringsToAddresses(hexStrings []string) []common.Address { - var addresses []common.Address - for _, hexString := range hexStrings { - address := common.HexToAddress(hexString) - addresses = append(addresses, address) + // Search for _any_ of the topics in topic0 position; see docs on `FilterQuery` + Topics: [][]common.Hash{topic0s}, } - return addresses + logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) + if err != nil { + // TODO review aggregate fetching error handling + return []types.Log{}, err + } + + return logs, nil } diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index 73e82d6f..ae036206 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -47,205 +47,235 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog" ) +// Custom event transformers var ( - BiteTransformerInitializer = factories.Transformer{ + BiteTransformer = factories.Transformer{ Config: bite.BiteConfig, Converter: &bite.BiteConverter{}, Repository: &bite.BiteRepository{}, - }.NewTransformer + } - CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{ - Config: chop_lump.CatFileChopLumpConfig, - Converter: &chop_lump.CatFileChopLumpConverter{}, - Repository: &chop_lump.CatFileChopLumpRepository{}, - }.NewLogNoteTransformer - - CatFileFlipTransformerInitializer = factories.LogNoteTransformer{ - Config: flip.CatFileFlipConfig, - Converter: &flip.CatFileFlipConverter{}, - Repository: &flip.CatFileFlipRepository{}, - }.NewLogNoteTransformer - - CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{ - Config: pit_vow.CatFilePitVowConfig, - Converter: &pit_vow.CatFilePitVowConverter{}, - Repository: &pit_vow.CatFilePitVowRepository{}, - }.NewLogNoteTransformer - - DealTransformerInitializer = factories.LogNoteTransformer{ - Config: deal.DealConfig, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - }.NewLogNoteTransformer - - DentTransformerInitializer = factories.LogNoteTransformer{ - Config: dent.DentConfig, - Converter: &dent.DentConverter{}, - Repository: &dent.DentRepository{}, - }.NewLogNoteTransformer - - DripDripTransformerInitializer = factories.LogNoteTransformer{ - Config: drip_drip.DripDripConfig, - Converter: &drip_drip.DripDripConverter{}, - Repository: &drip_drip.DripDripRepository{}, - }.NewLogNoteTransformer - - DripFileIlkTransformerInitializer = factories.LogNoteTransformer{ - Config: ilk2.DripFileIlkConfig, - Converter: &ilk2.DripFileIlkConverter{}, - Repository: &ilk2.DripFileIlkRepository{}, - }.NewLogNoteTransformer - - DripFileRepoTransformerInitializer = factories.LogNoteTransformer{ - Config: repo.DripFileRepoConfig, - Converter: &repo.DripFileRepoConverter{}, - Repository: &repo.DripFileRepoRepository{}, - }.NewLogNoteTransformer - - DripFileVowTransfromerInitializer = factories.LogNoteTransformer{ - Config: vow.DripFileVowConfig, - Converter: &vow.DripFileVowConverter{}, - Repository: &vow.DripFileVowRepository{}, - }.NewLogNoteTransformer - - FlapKickTransformerInitializer = factories.Transformer{ + FlapKickTransformer = factories.Transformer{ Config: flap_kick.FlapKickConfig, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, - }.NewTransformer + } - FlipKickTransformerInitializer = factories.Transformer{ + FlipKickTransformer = factories.Transformer{ Config: flip_kick.FlipKickConfig, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, - }.NewTransformer + } - FlogTransformerInitializer = factories.LogNoteTransformer{ - Config: vow_flog.VowFlogConfig, - Converter: &vow_flog.VowFlogConverter{}, - Repository: &vow_flog.VowFlogRepository{}, - }.NewLogNoteTransformer - - FrobTransformerInitializer = factories.Transformer{ + FrobTransformer = factories.Transformer{ Config: frob.FrobConfig, Converter: &frob.FrobConverter{}, Repository: &frob.FrobRepository{}, - }.NewTransformer + } - FlopKickTransformerInitializer = factories.Transformer{ + FlopKickTransformer = factories.Transformer{ Config: flop_kick.Config, Converter: &flop_kick.FlopKickConverter{}, Repository: &flop_kick.FlopKickRepository{}, - }.NewTransformer + } - PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{ + customEventTransformers = []factories.Transformer { + BiteTransformer, + FlapKickTransformer, + FlipKickTransformer, + FrobTransformer, + FlopKickTransformer, + } +) + +// LogNote transformers +var ( + CatFileChopLumpTransformer = factories.LogNoteTransformer{ + Config: chop_lump.CatFileChopLumpConfig, + Converter: &chop_lump.CatFileChopLumpConverter{}, + Repository: &chop_lump.CatFileChopLumpRepository{}, + } + + CatFileFlipTransformer = factories.LogNoteTransformer{ + Config: flip.CatFileFlipConfig, + Converter: &flip.CatFileFlipConverter{}, + Repository: &flip.CatFileFlipRepository{}, + } + + CatFilePitVowTransformer = factories.LogNoteTransformer{ + Config: pit_vow.CatFilePitVowConfig, + Converter: &pit_vow.CatFilePitVowConverter{}, + Repository: &pit_vow.CatFilePitVowRepository{}, + } + + DealTransformer = factories.LogNoteTransformer{ + Config: deal.DealConfig, + Converter: &deal.DealConverter{}, + Repository: &deal.DealRepository{}, + } + + DentTransformer = factories.LogNoteTransformer{ + Config: dent.DentConfig, + Converter: &dent.DentConverter{}, + Repository: &dent.DentRepository{}, + } + + DripDripTransformer = factories.LogNoteTransformer{ + Config: drip_drip.DripDripConfig, + Converter: &drip_drip.DripDripConverter{}, + Repository: &drip_drip.DripDripRepository{}, + } + + DripFileIlkTransformer = factories.LogNoteTransformer{ + Config: ilk2.DripFileIlkConfig, + Converter: &ilk2.DripFileIlkConverter{}, + Repository: &ilk2.DripFileIlkRepository{}, + } + + DripFileRepoTransformer = factories.LogNoteTransformer{ + Config: repo.DripFileRepoConfig, + Converter: &repo.DripFileRepoConverter{}, + Repository: &repo.DripFileRepoRepository{}, + } + + DripFileVowTransfromer = factories.LogNoteTransformer{ + Config: vow.DripFileVowConfig, + Converter: &vow.DripFileVowConverter{}, + Repository: &vow.DripFileVowRepository{}, + } + + FlogTransformer = factories.LogNoteTransformer{ + Config: vow_flog.VowFlogConfig, + Converter: &vow_flog.VowFlogConverter{}, + Repository: &vow_flog.VowFlogRepository{}, + } + + PitFileDebtCeilingTransformer = factories.LogNoteTransformer{ Config: debt_ceiling.DebtCeilingFileConfig, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, - }.NewLogNoteTransformer + } - PitFileIlkTransformerInitializer = factories.LogNoteTransformer{ + PitFileIlkTransformer = factories.LogNoteTransformer{ Config: ilk.IlkFileConfig, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, - }.NewLogNoteTransformer + } - PriceFeedTransformerInitializer = factories.LogNoteTransformer{ + PriceFeedTransformer = factories.LogNoteTransformer{ Config: price_feeds.PriceFeedConfig, Converter: &price_feeds.PriceFeedConverter{}, Repository: &price_feeds.PriceFeedRepository{}, - }.NewLogNoteTransformer + } - TendTransformerInitializer = factories.LogNoteTransformer{ + TendTransformer = factories.LogNoteTransformer{ Config: tend.TendConfig, Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, - }.NewLogNoteTransformer + } - VatInitTransformerInitializer = factories.LogNoteTransformer{ + VatInitTransformer = factories.LogNoteTransformer{ Config: vat_init.VatInitConfig, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, - }.NewLogNoteTransformer + } - VatGrabTransformerInitializer = factories.LogNoteTransformer{ + VatGrabTransformer = factories.LogNoteTransformer{ Config: vat_grab.VatGrabConfig, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, - }.NewLogNoteTransformer + } - VatFoldTransformerInitializer = factories.LogNoteTransformer{ + VatFoldTransformer = factories.LogNoteTransformer{ Config: vat_fold.VatFoldConfig, Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, - }.NewLogNoteTransformer + } - VatHealTransformerInitializer = factories.LogNoteTransformer{ + VatHealTransformer = factories.LogNoteTransformer{ Config: vat_heal.VatHealConfig, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, - }.NewLogNoteTransformer + } - VatMoveTransformerInitializer = factories.LogNoteTransformer{ + VatMoveTransformer = factories.LogNoteTransformer{ Config: vat_move.VatMoveConfig, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, - }.NewLogNoteTransformer + } - VatSlipTransformerInitializer = factories.LogNoteTransformer{ + VatSlipTransformer = factories.LogNoteTransformer{ Config: vat_slip.VatSlipConfig, Converter: &vat_slip.VatSlipConverter{}, Repository: &vat_slip.VatSlipRepository{}, - }.NewLogNoteTransformer + } - VatTollTransformerInitializer = factories.LogNoteTransformer{ + VatTollTransformer = factories.LogNoteTransformer{ Config: vat_toll.VatTollConfig, Converter: &vat_toll.VatTollConverter{}, Repository: &vat_toll.VatTollRepository{}, - }.NewLogNoteTransformer + } - VatTuneTransformerInitializer = factories.LogNoteTransformer{ + VatTuneTransformer = factories.LogNoteTransformer{ Config: vat_tune.VatTuneConfig, Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, - }.NewLogNoteTransformer + } - VatFluxTransformerInitializer = factories.LogNoteTransformer{ + VatFluxTransformer = factories.LogNoteTransformer{ Config: vat_flux.VatFluxConfig, Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, - }.NewLogNoteTransformer + } + + logNoteTransformers = []factories.LogNoteTransformer { + CatFileChopLumpTransformer, + CatFileFlipTransformer, + CatFilePitVowTransformer, + DealTransformer, + DentTransformer, + DripDripTransformer, + DripFileIlkTransformer, + DripFileRepoTransformer, + DripFileVowTransfromer, + FlogTransformer, + PitFileDebtCeilingTransformer, + PitFileIlkTransformer, + PriceFeedTransformer, + TendTransformer, + VatInitTransformer, + VatGrabTransformer, + VatFoldTransformer, + VatHealTransformer, + VatMoveTransformer, + VatSlipTransformer, + VatTollTransformer, + VatTuneTransformer, + VatFluxTransformer, + } ) -func TransformerInitializers() []shared.TransformerInitializer { - return []shared.TransformerInitializer{ - BiteTransformerInitializer, - CatFileChopLumpTransformerInitializer, - CatFileFlipTransformerInitializer, - CatFilePitVowTransformerInitializer, - DealTransformerInitializer, - DentTransformerInitializer, - DripDripTransformerInitializer, - DripFileIlkTransformerInitializer, - DripFileVowTransfromerInitializer, - DripFileRepoTransformerInitializer, - FlapKickTransformerInitializer, - FlipKickTransformerInitializer, - FlogTransformerInitializer, - FlopKickTransformerInitializer, - FrobTransformerInitializer, - PitFileDebtCeilingTransformerInitializer, - PitFileIlkTransformerInitializer, - PriceFeedTransformerInitializer, - TendTransformerInitializer, - VatGrabTransformerInitializer, - VatInitTransformerInitializer, - VatMoveTransformerInitializer, - VatHealTransformerInitializer, - VatFoldTransformerInitializer, - VatSlipTransformerInitializer, - VatTollTransformerInitializer, - VatTuneTransformerInitializer, - VatFluxTransformerInitializer, +// `TransformerInitializers` returns a list of functions, that given a db pointer +// will return a `shared.Transformer` +func TransformerInitializers() (initializers []shared.TransformerInitializer) { + for _, transformer := range logNoteTransformers { + initializers = append(initializers, transformer.NewLogNoteTransformer) } + + for _, transformer := range customEventTransformers { + initializers = append(initializers, transformer.NewTransformer) + } + return +} + +// `TransformerConfigs` returns the config structs for all available transformers +func TransformerConfigs() (allConfigs []shared.TransformerConfig) { + for _, transformer := range logNoteTransformers { + allConfigs = append(allConfigs, transformer.Config) + } + + for _, transformer := range logNoteTransformers { + allConfigs = append(allConfigs, transformer.Config) + } + return } From 38c745e8c3113a27e68f7cbe82799d169f5acf02 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 4 Dec 2018 16:04:13 +0100 Subject: [PATCH 06/33] Refactor fetching out from repositories to log_fetcher and watcher --- libraries/shared/watcher.go | 9 +- pkg/transformers/bite/repository.go | 5 -- .../cat_file/chop_lump/repository.go | 5 -- pkg/transformers/cat_file/flip/repository.go | 5 -- .../cat_file/pit_vow/repository.go | 5 -- pkg/transformers/deal/repository.go | 5 -- pkg/transformers/dent/repository.go | 5 -- pkg/transformers/drip_drip/repository.go | 5 -- pkg/transformers/drip_file/ilk/repository.go | 5 -- pkg/transformers/drip_file/repo/repository.go | 5 -- pkg/transformers/drip_file/vow/repository.go | 5 -- pkg/transformers/factories/repository.go | 2 - pkg/transformers/flap_kick/repository.go | 5 -- pkg/transformers/flip_kick/repository.go | 5 -- pkg/transformers/flop_kick/repository.go | 5 -- pkg/transformers/frob/repository.go | 5 -- .../pit_file/debt_ceiling/repository.go | 5 -- pkg/transformers/pit_file/ilk/repository.go | 5 -- pkg/transformers/price_feeds/repository.go | 5 -- pkg/transformers/shared/log_fetcher_test.go | 12 ++- pkg/transformers/shared/repository.go | 61 ++++++++++++- .../shared/repository_utility_test.go | 90 +++++++++++++++++++ pkg/transformers/tend/repository.go | 5 -- pkg/transformers/vat_flux/repository.go | 5 -- pkg/transformers/vat_fold/repository.go | 5 -- pkg/transformers/vat_grab/repository.go | 5 -- pkg/transformers/vat_heal/repository.go | 5 -- pkg/transformers/vat_init/repository.go | 5 -- pkg/transformers/vat_move/repository.go | 5 -- pkg/transformers/vat_slip/repository.go | 5 -- pkg/transformers/vat_toll/repository.go | 5 -- pkg/transformers/vat_tune/repository.go | 5 -- pkg/transformers/vow_flog/repository.go | 5 -- 33 files changed, 163 insertions(+), 151 deletions(-) create mode 100644 pkg/transformers/shared/repository_utility_test.go diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 2820ad1d..ff138dfc 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -51,9 +51,14 @@ func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { } func (watcher *Watcher) Execute() error { - // TODO Solve checkedHeadersColumn issue + checkedColumnNames, err := shared.GetCheckedColumnNames(&watcher.DB) + if err != nil { + return err + } + notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) + // TODO Handle start and end numbers in transformers? - missingHeaders, err := shared.MissingHeaders(0, -1, &watcher.DB, "") + missingHeaders, err := shared.MissingHeaders(0, -1, &watcher.DB, notCheckedSQL) for _, header := range missingHeaders { // TODO Extend FetchLogs for doing several blocks at a time diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 54cdad84..1e3d860b 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -16,7 +16,6 @@ package bite import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -65,7 +64,3 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) } - -func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked) -} diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index 52ef1446..9dff41f7 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -16,7 +16,6 @@ package chop_lump import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) } -func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked) -} - func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/flip/repository.go b/pkg/transformers/cat_file/flip/repository.go index 8f59c20f..a07ca207 100644 --- a/pkg/transformers/cat_file/flip/repository.go +++ b/pkg/transformers/cat_file/flip/repository.go @@ -16,7 +16,6 @@ package flip import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) } -func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked) -} - func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/pit_vow/repository.go b/pkg/transformers/cat_file/pit_vow/repository.go index 6aa7a108..861dc2a8 100644 --- a/pkg/transformers/cat_file/pit_vow/repository.go +++ b/pkg/transformers/cat_file/pit_vow/repository.go @@ -16,7 +16,6 @@ package pit_vow import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) } -func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked) -} - func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go index 84a40325..06f23230 100644 --- a/pkg/transformers/deal/repository.go +++ b/pkg/transformers/deal/repository.go @@ -16,7 +16,6 @@ package deal import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) } -func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked) -} - func (repository *DealRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index e237fb54..0ff5370b 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -16,7 +16,6 @@ package dent import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -67,10 +66,6 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) } -func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked) -} - func (repository *DentRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index 95780c74..edb1208a 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -16,7 +16,6 @@ package drip_drip import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) } -func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked) -} - func (repository *DripDripRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 8c602ef9..117a28fd 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -16,7 +16,6 @@ package ilk import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) } -func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked) -} - func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 592e92e6..dc9589a7 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -16,7 +16,6 @@ package repo import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) } -func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked) -} - func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 35b089d0..aa23bea8 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -16,7 +16,6 @@ package vow import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -63,10 +62,6 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) } -func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked) -} - func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/factories/repository.go b/pkg/transformers/factories/repository.go index 015fe49f..0ddc9274 100644 --- a/pkg/transformers/factories/repository.go +++ b/pkg/transformers/factories/repository.go @@ -15,13 +15,11 @@ package factories import ( - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) type Repository interface { Create(headerID int64, models []interface{}) error MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) SetDB(db *postgres.DB) } diff --git a/pkg/transformers/flap_kick/repository.go b/pkg/transformers/flap_kick/repository.go index 59f5803a..f9f11c7a 100644 --- a/pkg/transformers/flap_kick/repository.go +++ b/pkg/transformers/flap_kick/repository.go @@ -16,7 +16,6 @@ package flap_kick import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -60,10 +59,6 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) } -func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked) -} - func (repository *FlapKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flip_kick/repository.go b/pkg/transformers/flip_kick/repository.go index 21de65bc..db8a8d01 100644 --- a/pkg/transformers/flip_kick/repository.go +++ b/pkg/transformers/flip_kick/repository.go @@ -17,7 +17,6 @@ package flip_kick import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -60,10 +59,6 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) } -func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked) -} - func (repository *FlipKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flop_kick/repository.go b/pkg/transformers/flop_kick/repository.go index b175caf9..7a302c75 100644 --- a/pkg/transformers/flop_kick/repository.go +++ b/pkg/transformers/flop_kick/repository.go @@ -16,7 +16,6 @@ package flop_kick import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) } -func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked) -} - func (repository *FlopKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index afe7c4b1..bf6d4d08 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -17,7 +17,6 @@ package frob import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -59,10 +58,6 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) } -func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked) -} - func (repository *FrobRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 34e398f8..12f69fb2 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -16,7 +16,6 @@ package debt_ceiling import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) } -func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked) -} - func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index c186a638..9a187de7 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -16,7 +16,6 @@ package ilk import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) } -func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked) -} - func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go index 59dbab8c..cbb9a012 100644 --- a/pkg/transformers/price_feeds/repository.go +++ b/pkg/transformers/price_feeds/repository.go @@ -16,7 +16,6 @@ package price_feeds import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -58,10 +57,6 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) } -func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked) -} - func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/shared/log_fetcher_test.go b/pkg/transformers/shared/log_fetcher_test.go index 8e5aa7fa..4a0c55fa 100644 --- a/pkg/transformers/shared/log_fetcher_test.go +++ b/pkg/transformers/shared/log_fetcher_test.go @@ -32,8 +32,12 @@ var _ = Describe("Fetcher", func() { fetcher := shared.NewFetcher(blockChain) header := fakes.FakeHeader - addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"} - topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}} + addresses := []common.Address{ + common.HexToAddress("0xfakeAddress"), + common.HexToAddress("0xanotherFakeAddress"), + } + + topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})} _, err := fetcher.FetchLogs(addresses, topicZeros, header) @@ -45,7 +49,7 @@ var _ = Describe("Fetcher", func() { expectedQuery := ethereum.FilterQuery{ BlockHash: &blockHash, Addresses: []common.Address{address1, address2}, - Topics: topicZeros, + Topics: [][]common.Hash{topicZeros}, } blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) }) @@ -55,7 +59,7 @@ var _ = Describe("Fetcher", func() { blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) fetcher := shared.NewFetcher(blockChain) - _, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{}) + _, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{}) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index d0c6b5f6..a1d5c8db 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -2,8 +2,11 @@ package shared import ( "database/sql" + "database/sql/driver" + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "strings" ) func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { @@ -22,7 +25,9 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo return err } -func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) { +// Treats a header as missing if it's not in the headers table, or not checked for some log type +// TODO Revisit definition of "checked header +func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { var result []core.Header var query string var err error @@ -30,14 +35,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D if endingBlockNumber == -1 { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) + WHERE (header_id ISNULL OR ` + notCheckedSQL + `) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2` err = db.Select(&result, query, startingBlockNumber, db.Node.ID) } else { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) + WHERE (header_id ISNULL OR ` + notCheckedSQL + `) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3` @@ -46,3 +51,53 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D return result, err } + +func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { + // Query returns `[]driver.Value`, nullable polymorphic interface + var queryResult []driver.Value + columnNamesQuery := + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'checked_headers' + AND column_name != 'id' + AND column_name != 'header_id';` + + err := db.Select(&queryResult, columnNamesQuery) + if err != nil { + return []string{}, err + } + + // Transform column names from `driver.Value` to strings + var columnNames []string + for _, result := range queryResult { + if columnName, ok := result.(string); ok { + columnNames = append(columnNames, columnName) + } else { + return []string{}, fmt.Errorf("incorrect value for checked_headers column name") + } + } + + return columnNames, nil +} + +// Builds a SQL string that checks if any column value is FALSE, given the column names. +// Defaults to FALSE when no columns are provided. +// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" +// [] => "FALSE" +func CreateNotCheckedSQL(boolColumns []string) string { + var result strings.Builder + + if len(boolColumns) == 0 { + return "FALSE" + } + + result.WriteString("NOT (") + for _, column := range boolColumns[:len(boolColumns)-1] { + result.WriteString(fmt.Sprintf("%v AND ", column)) + } + + // No trailing "OR" for last column name + result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1])) + + return result.String() +} diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go new file mode 100644 index 00000000..e07f939b --- /dev/null +++ b/pkg/transformers/shared/repository_utility_test.go @@ -0,0 +1,90 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/test_config" +) + +var _ = Describe("Repository utilities", func() { + Describe("GetCheckedColumnNames", func() { + It("gets the column names from checked_headers", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + expectedColumnNames := getExpectedColumnNames() + actualColumnNames, err := shared.GetCheckedColumnNames(db) + Expect(err).NotTo(HaveOccurred()) + Expect(actualColumnNames).To(Equal(expectedColumnNames)) + }) + }) + + Describe("CreateNotCheckedSQL", func() { + It("generates a correct SQL string for one column", func() { + columns := []string{"columnA"} + expected := "NOT (columnA)" + actual := shared.CreateNotCheckedSQL(columns) + Expect(actual).To(Equal(expected)) + }) + + It("generates a correct SQL string for several columns", func() { + columns := []string{"columnA", "columnB"} + expected := "NOT (columnA AND columnB)" + actual := shared.CreateNotCheckedSQL(columns) + Expect(actual).To(Equal(expected)) + }) + + It("defaults to FALSE when there are no columns", func() { + expected := "FALSE" + actual := shared.CreateNotCheckedSQL([]string{}) + Expect(actual).To(Equal(expected)) + }) + }) +}) + +func getExpectedColumnNames() []string { + return []string{ + "price_feeds_checked", + "flip_kick_checked", + "frob_checked", + "tend_checked", + "bite_checked", + "dent_checked", + "pit_file_debt_ceiling_checked", + "pit_file_ilk_checked", + "vat_init_checked", + "drip_file_ilk_checked", + "drip_file_repo_checked", + "drip_file_vow_checked", + "deal_checked", + "drip_drip_checked", + "cat_file_chop_lump_checked", + "cat_file_flip_checked", + "cat_file_pit_vow_checked", + "flop_kick_checked", + "vat_move_checked", + "vat_fold_checked", + "vat_heal_checked", + "vat_toll_checked", + "vat_tune_checked", + "vat_grab_checked", + "vat_flux_checked", + "vat_slip_checked", + "vow_flog_checked", + "flap_kick_checked", + } +} \ No newline at end of file diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 61982e12..17620a00 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -16,7 +16,6 @@ package tend import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -68,10 +67,6 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) } -func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked) -} - func (repository *TendRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index d87ffca7..2febe527 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -16,7 +16,6 @@ package vat_flux import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository VatFluxRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) } -func (repository VatFluxRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatFluxChecked) -} - func (repository *VatFluxRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index bb14f172..5d831bca 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -17,7 +17,6 @@ package vat_fold import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -63,10 +62,6 @@ func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) } -func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatFoldChecked) -} - func (repository *VatFoldRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index b826b7c1..a8a274a4 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -3,7 +3,6 @@ package vat_grab import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -47,10 +46,6 @@ func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) } -func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatGrabChecked) -} - func (repository *VatGrabRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_heal/repository.go b/pkg/transformers/vat_heal/repository.go index 5050d6d6..aed8fcd0 100644 --- a/pkg/transformers/vat_heal/repository.go +++ b/pkg/transformers/vat_heal/repository.go @@ -17,7 +17,6 @@ package vat_heal import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) return tx.Commit() } -func (repository VatHealRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatHealChecked) -} - func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) } diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index cee9e44a..14b2d4c8 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -16,7 +16,6 @@ package vat_init import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -63,10 +62,6 @@ func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) } -func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatInitChecked) -} - func (repository *VatInitRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index f47db4d6..2548c2b4 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -16,7 +16,6 @@ package vat_move import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -60,10 +59,6 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) return tx.Commit() } -func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatMoveChecked) -} - func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) } diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index 1c6b8d06..15f412ca 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -2,7 +2,6 @@ package vat_slip import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -48,10 +47,6 @@ func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) } -func (repository VatSlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatSlipChecked) -} - func (repository *VatSlipRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index 39371193..2c9be8a4 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -2,7 +2,6 @@ package vat_toll import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -47,10 +46,6 @@ func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) } -func (repository VatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTollChecked) -} - func (repository *VatTollRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index b7298cce..c9951ce6 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -2,7 +2,6 @@ package vat_tune import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -47,10 +46,6 @@ func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) } -func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked) -} - func (repository *VatTuneRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vow_flog/repository.go b/pkg/transformers/vow_flog/repository.go index 0754fd5f..48076802 100644 --- a/pkg/transformers/vow_flog/repository.go +++ b/pkg/transformers/vow_flog/repository.go @@ -16,7 +16,6 @@ package vow_flog import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository VowFlogRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) } -func (repository VowFlogRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VowFlogChecked) -} - func (repository *VowFlogRepository) SetDB(db *postgres.DB) { repository.db = db } From 8bebcdc06412d3bb5a69b1c51723eab3f0c5c1f2 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 4 Dec 2018 17:05:34 +0100 Subject: [PATCH 07/33] Change transformer.Execute to single header --- libraries/shared/watcher.go | 2 +- .../factories/log_note_transformer.go | 38 ++++++++-------- pkg/transformers/factories/transformer.go | 43 ++++++++----------- pkg/transformers/shared/transformer.go | 2 +- 4 files changed, 38 insertions(+), 47 deletions(-) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index ff138dfc..70277cfd 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -74,7 +74,7 @@ func (watcher *Watcher) Execute() error { // TODO delegate log chunks to respective transformers // Need to get the transformer name... :/ logChunk := chunkedLogs["transformerName"] - err = transformer.Execute(logChunk, missingHeaders) + err = transformer.Execute(logChunk, header) } } return err diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index 453d72b0..663b9dc0 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -34,32 +34,28 @@ func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) sha return transformer } -func (transformer LogNoteTransformer) Execute(logs []types.Log, missingHeaders []core.Header) error { +func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error { transformerName := transformer.Config.TransformerName - for _, header := range missingHeaders { - // No matching logs, mark the header as checked for this type of logs - if len(logs) < 1 { - err := transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - log.Printf("Error marking header as checked in %v: %v", transformerName, err) - return err - } - // Continue with the next header; nothing to persist - continue - } - - models, err := transformer.Converter.ToModels(logs) + // No matching logs, mark the header as checked for this type of logs + if len(logs) < 1 { + err := transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { - log.Printf("Error converting logs in %v: %v", transformerName, err) - return err - } - - err = transformer.Repository.Create(header.Id, models) - if err != nil { - log.Printf("Error persisting %v record: %v", transformerName, err) + log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err } } + + models, err := transformer.Converter.ToModels(logs) + if err != nil { + log.Printf("Error converting logs in %v: %v", transformerName, err) + return err + } + + err = transformer.Repository.Create(header.Id, models) + if err != nil { + log.Printf("Error persisting %v record: %v", transformerName, err) + return err + } return nil } diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 6c1a8a4f..28ed3953 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -34,39 +34,34 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transforme return transformer } -func (transformer Transformer) Execute(logs []types.Log, missingHeaders []core.Header) error { +func (transformer Transformer) Execute(logs []types.Log, header core.Header) error { transformerName := transformer.Config.TransformerName config := transformer.Config - for _, header := range missingHeaders { - if len(logs) < 1 { - err := transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - log.Printf("Error marking header as checked in %v: %v", transformerName, err) - return err - } - - continue - } - - entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs) + if len(logs) < 1 { + err := transformer.Repository.MarkHeaderChecked(header.Id) if err != nil { - log.Printf("Error converting logs to entities in %v: %v", transformerName, err) + log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err } + } - models, err := transformer.Converter.ToModels(entities) - if err != nil { - log.Printf("Error converting entities to models in %v: %v", transformerName, err) - return err - } + entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs) + if err != nil { + log.Printf("Error converting logs to entities in %v: %v", transformerName, err) + return err + } - err = transformer.Repository.Create(header.Id, models) - if err != nil { - log.Printf("Error persisting %v record: %v", transformerName, err) - return err - } + models, err := transformer.Converter.ToModels(entities) + if err != nil { + log.Printf("Error converting entities to models in %v: %v", transformerName, err) + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + log.Printf("Error persisting %v record: %v", transformerName, err) + return err } return nil diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 0e5a3f9e..8b5c8c49 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -23,7 +23,7 @@ import ( ) type Transformer interface { - Execute(logs []types.Log, missingHeaders []core.Header) error + Execute(logs []types.Log, header core.Header) error } type TransformerInitializer func(db *postgres.DB) Transformer From 9789648cc2a65214a6c31880c6bdef3bdb1e3356 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 4 Dec 2018 17:40:39 +0100 Subject: [PATCH 08/33] Delegate log chunks to respective transformers --- libraries/shared/watcher.go | 4 +--- pkg/transformers/factories/log_note_transformer.go | 4 ++++ pkg/transformers/factories/transformer.go | 4 ++++ pkg/transformers/shared/repository.go | 1 - pkg/transformers/shared/transformer.go | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 70277cfd..3b1cff20 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -71,9 +71,7 @@ func (watcher *Watcher) Execute() error { chunkedLogs := watcher.Chunker.ChunkLogs(logs) for _, transformer := range watcher.Transformers { - // TODO delegate log chunks to respective transformers - // Need to get the transformer name... :/ - logChunk := chunkedLogs["transformerName"] + logChunk := chunkedLogs[transformer.Name()] err = transformer.Execute(logChunk, header) } } diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index 663b9dc0..849d9619 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -59,3 +59,7 @@ func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Head } return nil } + +func (transformer LogNoteTransformer) Name() string { + return transformer.Config.TransformerName +} diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 28ed3953..eae4fd7f 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -66,3 +66,7 @@ func (transformer Transformer) Execute(logs []types.Log, header core.Header) err return nil } + +func (transformer Transformer) Name() string { + return transformer.Config.TransformerName +} diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index a1d5c8db..0380d50a 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -26,7 +26,6 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo } // Treats a header as missing if it's not in the headers table, or not checked for some log type -// TODO Revisit definition of "checked header func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { var result []core.Header var query string diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 8b5c8c49..a5ff5835 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -24,6 +24,7 @@ import ( type Transformer interface { Execute(logs []types.Log, header core.Header) error + Name() string } type TransformerInitializer func(db *postgres.DB) Transformer From 47c75d055ba24e1fdb217de75870cf368fa7f6a6 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 4 Dec 2018 17:53:46 +0100 Subject: [PATCH 09/33] Update repository tests for absence of MissingHeaders --- pkg/transformers/bite/repository_test.go | 10 ---------- pkg/transformers/cat_file/chop_lump/repository_test.go | 9 --------- pkg/transformers/cat_file/flip/repository_test.go | 8 -------- pkg/transformers/cat_file/pit_vow/repository_test.go | 9 --------- pkg/transformers/deal/repository_test.go | 9 --------- pkg/transformers/dent/repository_test.go | 9 --------- pkg/transformers/drip_drip/repository_test.go | 9 --------- pkg/transformers/drip_file/ilk/repository_test.go | 9 --------- pkg/transformers/drip_file/repo/repository_test.go | 9 --------- pkg/transformers/drip_file/vow/repository_test.go | 9 --------- pkg/transformers/flap_kick/repository_test.go | 9 --------- pkg/transformers/flip_kick/repository_test.go | 9 --------- pkg/transformers/flop_kick/repository_test.go | 9 --------- pkg/transformers/frob/repository_test.go | 9 --------- .../pit_file/debt_ceiling/repository_test.go | 9 --------- pkg/transformers/pit_file/ilk/repository_test.go | 9 --------- pkg/transformers/price_feeds/repository_test.go | 9 --------- pkg/transformers/tend/repository_test.go | 9 --------- .../test_data/shared_behaviors/repository_behaviors.go | 7 +++---- pkg/transformers/vat_flux/repository_test.go | 9 --------- pkg/transformers/vat_fold/repository_test.go | 9 --------- pkg/transformers/vat_grab/repository_test.go | 9 --------- pkg/transformers/vat_heal/repository_test.go | 9 --------- pkg/transformers/vat_init/repository_test.go | 9 --------- pkg/transformers/vat_move/repository_test.go | 9 --------- pkg/transformers/vat_slip/repository_test.go | 9 --------- pkg/transformers/vat_toll/repository_test.go | 9 --------- pkg/transformers/vat_tune/repository_test.go | 9 --------- pkg/transformers/vow_flog/repository_test.go | 9 --------- 29 files changed, 3 insertions(+), 256 deletions(-) diff --git a/pkg/transformers/bite/repository_test.go b/pkg/transformers/bite/repository_test.go index 480fe165..c8d48f68 100644 --- a/pkg/transformers/bite/repository_test.go +++ b/pkg/transformers/bite/repository_test.go @@ -86,14 +86,4 @@ var _ = Describe("Bite repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &biteRepository, - RepositoryTwo: &bite.BiteRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/cat_file/chop_lump/repository_test.go b/pkg/transformers/cat_file/chop_lump/repository_test.go index ec10826a..41041f49 100644 --- a/pkg/transformers/cat_file/chop_lump/repository_test.go +++ b/pkg/transformers/cat_file/chop_lump/repository_test.go @@ -98,13 +98,4 @@ var _ = Describe("Cat file chop lump repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &catFileRepository, - RepositoryTwo: &chop_lump.CatFileChopLumpRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/cat_file/flip/repository_test.go b/pkg/transformers/cat_file/flip/repository_test.go index c50e3a97..8dfe6027 100644 --- a/pkg/transformers/cat_file/flip/repository_test.go +++ b/pkg/transformers/cat_file/flip/repository_test.go @@ -83,12 +83,4 @@ var _ = Describe("Cat file flip repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &catFileFlipRepository, - RepositoryTwo: &flip.CatFileFlipRepository{}, - } - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/cat_file/pit_vow/repository_test.go b/pkg/transformers/cat_file/pit_vow/repository_test.go index 5f97cfe1..c7d31167 100644 --- a/pkg/transformers/cat_file/pit_vow/repository_test.go +++ b/pkg/transformers/cat_file/pit_vow/repository_test.go @@ -81,13 +81,4 @@ var _ = Describe("Cat file pit vow repository", func() { } shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &catFilePitVowRepository, - RepositoryTwo: &pit_vow.CatFilePitVowRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/deal/repository_test.go b/pkg/transformers/deal/repository_test.go index 5a58c4ca..2e3f02b0 100644 --- a/pkg/transformers/deal/repository_test.go +++ b/pkg/transformers/deal/repository_test.go @@ -84,13 +84,4 @@ var _ = Describe("Deal Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dealRepository, - RepositoryTwo: &deal.DealRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/dent/repository_test.go b/pkg/transformers/dent/repository_test.go index bad19213..b667228a 100644 --- a/pkg/transformers/dent/repository_test.go +++ b/pkg/transformers/dent/repository_test.go @@ -92,13 +92,4 @@ var _ = Describe("Dent Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dentRepository, - RepositoryTwo: &dent.DentRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_drip/repository_test.go b/pkg/transformers/drip_drip/repository_test.go index d66b9fd0..50646855 100644 --- a/pkg/transformers/drip_drip/repository_test.go +++ b/pkg/transformers/drip_drip/repository_test.go @@ -80,13 +80,4 @@ var _ = Describe("Drip drip repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripDripRepository, - RepositoryTwo: &drip_drip.DripDripRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index 6853f291..8dbada31 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Drip file ilk repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripFileIlkRepository, - RepositoryTwo: &ilk.DripFileIlkRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 8e297a3c..e8614b61 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Drip file repo repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripFileRepoRepository, - RepositoryTwo: &repo.DripFileRepoRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index 9ad62a49..71e8cd07 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -74,15 +74,6 @@ var _ = Describe("Drip file vow repository", func() { }) Describe("MarkHeaderChecked", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripFileVowRepository, - RepositoryTwo: &vow.DripFileVowRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - - Describe("MissingHeaders", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.DripFileVowChecked, Repository: &dripFileVowRepository, diff --git a/pkg/transformers/flap_kick/repository_test.go b/pkg/transformers/flap_kick/repository_test.go index 1012548b..2b401cfd 100644 --- a/pkg/transformers/flap_kick/repository_test.go +++ b/pkg/transformers/flap_kick/repository_test.go @@ -87,13 +87,4 @@ var _ = Describe("Flap Kick Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &flapKickRepository, - RepositoryTwo: &flap_kick.FlapKickRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/flip_kick/repository_test.go b/pkg/transformers/flip_kick/repository_test.go index 7ecb5cb0..6befc361 100644 --- a/pkg/transformers/flip_kick/repository_test.go +++ b/pkg/transformers/flip_kick/repository_test.go @@ -87,15 +87,6 @@ var _ = Describe("FlipKick Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("missing headers", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &flipKickRepository, - RepositoryTwo: &flip_kick.FlipKickRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) { diff --git a/pkg/transformers/flop_kick/repository_test.go b/pkg/transformers/flop_kick/repository_test.go index c916169d..8d5a8351 100644 --- a/pkg/transformers/flop_kick/repository_test.go +++ b/pkg/transformers/flop_kick/repository_test.go @@ -85,13 +85,4 @@ var _ = Describe("FlopRepository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &flop_kick.FlopKickRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/frob/repository_test.go b/pkg/transformers/frob/repository_test.go index 5d399edb..36c6036a 100644 --- a/pkg/transformers/frob/repository_test.go +++ b/pkg/transformers/frob/repository_test.go @@ -86,13 +86,4 @@ var _ = Describe("Frob repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &frobRepository, - RepositoryTwo: &frob.FrobRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index 06a7588b..026d0446 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -81,13 +81,4 @@ var _ = Describe("Pit file debt ceiling repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &pitFileDebtCeilingRepository, - RepositoryTwo: &debt_ceiling.PitFileDebtCeilingRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index b99f6ad5..fbcebbf2 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Pit file ilk repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &pitFileIlkRepository, - RepositoryTwo: &ilk.PitFileIlkRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/price_feeds/repository_test.go b/pkg/transformers/price_feeds/repository_test.go index 52765fd4..f58aef0d 100644 --- a/pkg/transformers/price_feeds/repository_test.go +++ b/pkg/transformers/price_feeds/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Price feeds repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &priceFeedRepository, - RepositoryTwo: &price_feeds.PriceFeedRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index dae2d4ba..0753524f 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -94,13 +94,4 @@ var _ = Describe("TendRepository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &tendRepository, - RepositoryTwo: &tend.TendRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go index 65ba206c..e6c24847 100644 --- a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go +++ b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go @@ -17,7 +17,6 @@ package shared_behaviors import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -25,7 +24,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" - "math/rand" ) var ( @@ -132,7 +130,8 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { }) } -func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInputs) { +// Saved, will base tests of aggregate fetching on this +/*func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInputs) { Describe("MissingHeaders", func() { var ( repository = inputs.Repository @@ -211,7 +210,7 @@ func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInput Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) }) }) -} +}*/ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehaviorInputs) { var repository = inputs.Repository diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go index 14b3f403..a8b7ab85 100644 --- a/pkg/transformers/vat_flux/repository_test.go +++ b/pkg/transformers/vat_flux/repository_test.go @@ -86,15 +86,6 @@ var _ = Describe("VatFlux Repository", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_flux.VatFluxRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkHeaderChecked", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatFluxChecked, diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index a26ff75b..fc551b37 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -84,13 +84,4 @@ var _ = Describe("Vat.fold repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_fold.VatFoldRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_grab/repository_test.go b/pkg/transformers/vat_grab/repository_test.go index 1d6e222c..e5dded85 100644 --- a/pkg/transformers/vat_grab/repository_test.go +++ b/pkg/transformers/vat_grab/repository_test.go @@ -73,13 +73,4 @@ var _ = Describe("Vat grab repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &vatGrabRepository, - RepositoryTwo: &vat_grab.VatGrabRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_heal/repository_test.go b/pkg/transformers/vat_heal/repository_test.go index 9cc668fc..d16bd9df 100644 --- a/pkg/transformers/vat_heal/repository_test.go +++ b/pkg/transformers/vat_heal/repository_test.go @@ -83,15 +83,6 @@ var _ = Describe("VatHeal Repository", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_heal.VatHealRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkCheckedHeader", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatHealChecked, diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index 93ca26fe..6e505171 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -72,15 +72,6 @@ var _ = Describe("Vat init repository", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_init.VatInitRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkHeaderChecked", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatInitChecked, diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index b4eb0bcc..179a2ee0 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -71,15 +71,6 @@ var _ = Describe("Vat Move", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_move.VatMoveRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkHeaderChecked", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatMoveChecked, diff --git a/pkg/transformers/vat_slip/repository_test.go b/pkg/transformers/vat_slip/repository_test.go index d3f1e64e..23a83150 100644 --- a/pkg/transformers/vat_slip/repository_test.go +++ b/pkg/transformers/vat_slip/repository_test.go @@ -68,13 +68,4 @@ var _ = Describe("Vat slip repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_slip.VatSlipRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_toll/repository_test.go b/pkg/transformers/vat_toll/repository_test.go index 6b3d21d4..3c2f9f0d 100644 --- a/pkg/transformers/vat_toll/repository_test.go +++ b/pkg/transformers/vat_toll/repository_test.go @@ -68,13 +68,4 @@ var _ = Describe("Vat toll repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_toll.VatTollRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_tune/repository_test.go b/pkg/transformers/vat_tune/repository_test.go index 5214f9e5..1700825b 100644 --- a/pkg/transformers/vat_tune/repository_test.go +++ b/pkg/transformers/vat_tune/repository_test.go @@ -71,13 +71,4 @@ var _ = Describe("Vat tune repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_tune.VatTuneRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vow_flog/repository_test.go b/pkg/transformers/vow_flog/repository_test.go index 153b1876..de6b3b94 100644 --- a/pkg/transformers/vow_flog/repository_test.go +++ b/pkg/transformers/vow_flog/repository_test.go @@ -80,13 +80,4 @@ var _ = Describe("Vow flog repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vow_flog.VowFlogRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) From e1ba7ac5b4dee88784bdc7da3e0a3bf6b08ea52f Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 7 Dec 2018 18:10:36 +0100 Subject: [PATCH 10/33] WIP update tests for aggregate fetching --- libraries/shared/watcher.go | 10 +- libraries/shared/watcher_test.go | 67 +++++++++---- .../repositories/header_repository.go | 1 - .../factories/log_note_transformer_test.go | 86 +++-------------- .../factories/transformer_test.go | 91 +++--------------- pkg/transformers/integration_tests/bite.go | 15 ++- .../integration_tests/cat_file.go | 47 +++++++--- pkg/transformers/integration_tests/deal.go | 63 ++++++++----- pkg/transformers/integration_tests/dent.go | 41 +++++--- .../integration_tests/drip_drip.go | 38 +++++--- .../integration_tests/drip_file_vow.go | 40 +++++--- .../integration_tests/flap_kick.go | 45 ++++++--- .../integration_tests/flip_kick.go | 19 ++-- .../integration_tests/flop_kick.go | 61 +++++++----- pkg/transformers/integration_tests/frob.go | 38 +++++--- pkg/transformers/integration_tests/helpers.go | 7 +- .../pit_file_debt_ceiling.go | 36 +++++-- .../integration_tests/pit_file_ilk.go | 16 +++- .../integration_tests/price_feeds.go | 94 ++++++++++--------- pkg/transformers/integration_tests/tend.go | 80 ++++++++-------- .../integration_tests/vat_flux.go | 16 +++- .../integration_tests/vat_fold.go | 40 +++++--- .../integration_tests/vat_grab.go | 15 ++- .../integration_tests/vat_heal.go | 14 ++- .../integration_tests/vat_init.go | 19 ++-- .../integration_tests/vat_move.go | 18 ++-- .../integration_tests/vat_slip.go | 22 +++-- .../integration_tests/vat_tune.go | 19 ++-- .../integration_tests/vow_flog.go | 40 +++++--- pkg/transformers/shared/log_chunker.go | 2 +- .../shared/repository_utility_test.go | 2 +- pkg/transformers/shared/transformer.go | 7 ++ .../test_data/mocks/repository.go | 17 ---- pkg/transformers/transformers.go | 4 +- 34 files changed, 631 insertions(+), 499 deletions(-) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 3b1cff20..89ee5a42 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -34,12 +34,12 @@ func NewWatcher(db postgres.DB, bc core.BlockChain) Watcher { fetcher := shared.NewFetcher(bc) return Watcher{ - DB: db, + DB: db, Blockchain: bc, - Fetcher: fetcher, - Chunker: chunker, - Addresses: contractAddresses, - Topics: topic0s, + Fetcher: fetcher, + Chunker: chunker, + Addresses: contractAddresses, + Topics: topic0s, } } diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index d9b1f55c..e6781d04 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -2,33 +2,49 @@ package shared_test import ( "errors" - + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/test_config" ) type MockTransformer struct { executeWasCalled bool executeError error + passedLogs []types.Log + passedHeader core.Header } -func (mh *MockTransformer) Execute() error { +func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { if mh.executeError != nil { return mh.executeError } mh.executeWasCalled = true + mh.passedLogs = logs + mh.passedHeader = header return nil } -func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared2.Transformer { +func (mh *MockTransformer) Name() string { + return "MockTransformer" +} + +func fakeTransformerInitializer(db *postgres.DB) shared2.Transformer { return &MockTransformer{} } var _ = Describe("Watcher", func() { + // TODO Add test for watcher setting the BC + // TODO Add tests for log chunk delegation + // TODO Add tests for aggregate fetching + // TODO Add tests for MissingHeaders + It("Adds transformers", func() { watcher := shared.Watcher{} @@ -47,24 +63,41 @@ var _ = Describe("Watcher", func() { Expect(len(watcher.Transformers)).To(Equal(2)) }) - It("Executes each transformer", func() { - watcher := shared.Watcher{} - fakeTransformer := &MockTransformer{} - watcher.Transformers = []shared2.Transformer{fakeTransformer} + Describe("with missing headers", func() { + var ( + db *postgres.DB + watcher shared.Watcher + fakeTransformer *MockTransformer + headerRepository repositories.HeaderRepository + ) - watcher.Execute() + BeforeEach(func() { + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + watcher = shared.NewWatcher(*db, nil) + headerRepository = repositories.NewHeaderRepository(db) + _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(err).NotTo(HaveOccurred()) + }) - Expect(fakeTransformer.executeWasCalled).To(BeTrue()) - }) + It("executes each transformer", func() { + fakeTransformer = &MockTransformer{} + watcher.Transformers = []shared2.Transformer{fakeTransformer} - It("Returns an error if transformer returns an error", func() { - watcher := shared.Watcher{} - fakeTransformer := &MockTransformer{executeError: errors.New("Something bad happened")} - watcher.Transformers = []shared2.Transformer{fakeTransformer} + err := watcher.Execute() - err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(fakeTransformer.executeWasCalled).To(BeTrue()) + }) - Expect(err).To(HaveOccurred()) - Expect(fakeTransformer.executeWasCalled).To(BeFalse()) + It("returns an error if transformer returns an error", func() { + fakeTransformer = &MockTransformer{executeError: errors.New("Something bad happened")} + watcher.Transformers = []shared2.Transformer{fakeTransformer} + + err := watcher.Execute() + + Expect(err).To(HaveOccurred()) + Expect(fakeTransformer.executeWasCalled).To(BeFalse()) + }) }) }) diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index c7948337..256eb9b9 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -3,7 +3,6 @@ package repositories import ( "database/sql" "errors" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) diff --git a/pkg/transformers/factories/log_note_transformer_test.go b/pkg/transformers/factories/log_note_transformer_test.go index d6eb2e24..678c9879 100644 --- a/pkg/transformers/factories/log_note_transformer_test.go +++ b/pkg/transformers/factories/log_note_transformer_test.go @@ -15,7 +15,7 @@ package factories_test import ( - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -31,9 +31,7 @@ var _ = Describe("LogNoteTransformer", func() { var ( repository mocks.MockRepository converter mocks.MockLogNoteConverter - fetcher mocks.MockLogFetcher headerOne core.Header - headerTwo core.Header transformer shared.Transformer model test_data.GenericModel config = test_data.GenericTestConfig @@ -43,75 +41,28 @@ var _ = Describe("LogNoteTransformer", func() { BeforeEach(func() { repository = mocks.MockRepository{} converter = mocks.MockLogNoteConverter{} - fetcher = mocks.MockLogFetcher{} transformer = factories.LogNoteTransformer{ Config: config, Converter: &converter, Repository: &repository, - Fetcher: &fetcher, - }.NewLogNoteTransformer(nil, nil) + }.NewLogNoteTransformer(nil) headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} - headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} }) - It("sets the blockchain and database", func() { - Expect(fetcher.SetBcCalled).To(BeTrue()) + It("sets the database", func() { Expect(repository.SetDbCalled).To(BeTrue()) }) - It("gets missing headers for block numbers specified in config", func() { - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) - }) - - It("returns error if repository returns error for missing headers", func() { - repository.SetMissingHeadersError(fakes.FakeError) - - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("fetches logs for missing headers", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) - expectedTopic := common.HexToHash(config.Topic) - Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{expectedTopic}})) - }) - - It("returns error if fetcher returns error", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetcherError(fakes.FakeError) - - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("marks header checked if no logs returned", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - - err := transformer.Execute() + It("marks header checked if no logs are provided", func() { + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.ToModelsCalledCounter).To(Equal(0)) @@ -119,30 +70,23 @@ var _ = Describe("LogNoteTransformer", func() { }) It("does not call repository.MarkCheckedHeader when there are logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) - - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedNotCalled() }) It("returns error if marking header checked returns err", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetMarkHeaderCheckedError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts matching logs to models", func() { - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) - - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.PassedLogs).To(Equal(logs)) @@ -150,20 +94,16 @@ var _ = Describe("LogNoteTransformer", func() { It("returns error if converter returns error", func() { converter.SetConverterError(fakes.FakeError) - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("persists the model", func() { - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) converter.SetReturnModels([]interface{}{model}) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) @@ -171,11 +111,9 @@ var _ = Describe("LogNoteTransformer", func() { }) It("returns error if repository returns error for create", func() { - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/factories/transformer_test.go b/pkg/transformers/factories/transformer_test.go index 929b7f8d..32d85880 100644 --- a/pkg/transformers/factories/transformer_test.go +++ b/pkg/transformers/factories/transformer_test.go @@ -15,7 +15,7 @@ package factories_test import ( - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -30,86 +30,39 @@ import ( var _ = Describe("Transformer", func() { var ( repository mocks.MockRepository - fetcher mocks.MockLogFetcher converter mocks.MockConverter transformer shared.Transformer headerOne core.Header - headerTwo core.Header config = test_data.GenericTestConfig logs = test_data.GenericTestLogs ) BeforeEach(func() { repository = mocks.MockRepository{} - fetcher = mocks.MockLogFetcher{} converter = mocks.MockConverter{} transformer = factories.Transformer{ Repository: &repository, - Fetcher: &fetcher, Converter: &converter, Config: config, - }.NewTransformer(nil, nil) + }.NewTransformer(nil) headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} - headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} }) - It("sets the blockchain and db", func() { - Expect(fetcher.SetBcCalled).To(BeTrue()) + It("sets the db", func() { Expect(repository.SetDbCalled).To(BeTrue()) }) - It("gets missing headers for blocks in the configured range", func() { - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) - }) - - It("returns an error if it fails to get missing headers", func() { - repository.SetMissingHeadersError(fakes.FakeError) - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("fetches eth logs for each missing header", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - expectedTopics := [][]common.Hash{{common.HexToHash(config.Topic)}} - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) - Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) - }) - - It("returns an error if fetching logs fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetcherError(fakes.FakeError) - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - It("marks header checked if no logs returned", func() { - headerID := int64(123) - repository.SetMissingHeaders([]core.Header{{Id: headerID}}) - - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) - repository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.ToEntitiesCalledCounter).To(Equal(0)) @@ -118,29 +71,23 @@ var _ = Describe("Transformer", func() { }) It("does not call repository.MarkCheckedHeader when there are logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) - - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedNotCalled() }) It("returns error if marking header checked returns err", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetMarkHeaderCheckedError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts an eth log to an entity", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.ContractAbi).To(Equal(config.ContractAbi)) @@ -148,45 +95,37 @@ var _ = Describe("Transformer", func() { }) It("returns an error if converter fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.ToEntitiesError = fakes.FakeError - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts an entity to a model", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{})) }) It("returns an error if converting to models fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.ToModelsError = fakes.FakeError - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("persists the record", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.ModelsToReturn = []interface{}{test_data.GenericModel{}} - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) @@ -194,10 +133,8 @@ var _ = Describe("Transformer", func() { }) It("returns error if persisting the record fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) repository.SetCreateError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/integration_tests/bite.go b/pkg/transformers/integration_tests/bite.go index 86c9f7ff..106f7c4f 100644 --- a/pkg/transformers/integration_tests/bite.go +++ b/pkg/transformers/integration_tests/bite.go @@ -45,17 +45,24 @@ var _ = Describe("Bite Transformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ Config: config, Converter: &bite.BiteConverter{}, Repository: &bite.BiteRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []bite.BiteModel diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 68983f88..059e37af 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "sort" @@ -39,6 +40,7 @@ var _ = Describe("Cat File transformer", func() { rpcClient client.RpcClient err error ethClient *ethclient.Client + fetcher shared.Fetcher ) BeforeEach(func() { @@ -48,13 +50,15 @@ var _ = Describe("Cat File transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + fetcher = shared.NewFetcher(blockChain) }) // Cat contract Kovan address: 0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0 It("persists a chop lump event", func() { // transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014 chopLumpBlockNumber := int64(8762253) - err = persistHeader(db, chopLumpBlockNumber, blockChain) + header, err := persistHeader(db, chopLumpBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := chop_lump.CatFileChopLumpConfig @@ -65,10 +69,16 @@ var _ = Describe("Cat File transformer", func() { Config: config, Converter: &chop_lump.CatFileChopLumpConverter{}, Repository: &chop_lump.CatFileChopLumpRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []chop_lump.CatFileChopLumpModel @@ -92,7 +102,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a flip event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe flipBlockNumber := int64(8751794) - err = persistHeader(db, flipBlockNumber, blockChain) + header, err := persistHeader(db, flipBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := flip.CatFileFlipConfig @@ -103,10 +113,17 @@ var _ = Describe("Cat File transformer", func() { Config: config, Converter: &flip.CatFileFlipConverter{}, Repository: &flip.CatFileFlipRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err := transformer.Execute() + + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flip.CatFileFlipModel @@ -122,7 +139,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a pit vow event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe pitVowBlockNumber := int64(8751794) - err = persistHeader(db, pitVowBlockNumber, blockChain) + header, err := persistHeader(db, pitVowBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := pit_vow.CatFilePitVowConfig @@ -133,10 +150,16 @@ var _ = Describe("Cat File transformer", func() { Config: config, Converter: &pit_vow.CatFilePitVowConverter{}, Repository: &pit_vow.CatFilePitVowRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []pit_vow.CatFilePitVowModel diff --git a/pkg/transformers/integration_tests/deal.go b/pkg/transformers/integration_tests/deal.go index a43e04cf..990b0d76 100644 --- a/pkg/transformers/integration_tests/deal.go +++ b/pkg/transformers/integration_tests/deal.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,8 +30,13 @@ import ( var _ = Describe("Deal transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + initializer factories.LogNoteTransformer + fetcher shared.Fetcher + addresses []common.Address + topics []common.Hash ) BeforeEach(func() { @@ -40,26 +46,35 @@ var _ = Describe("Deal transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = deal.DealConfig + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &deal.DealConverter{}, + Repository: &deal.DealRepository{}, + } + + fetcher = shared.NewFetcher(blockChain) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} + }) It("persists a flip deal log event", func() { // transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58 flipBlockNumber := int64(8958007) - err := persistHeader(db, flipBlockNumber, blockChain) + header, err := persistHeader(db, flipBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := deal.DealConfig - config.StartingBlockNumber = flipBlockNumber - config.EndingBlockNumber = flipBlockNumber + initializer.Config.StartingBlockNumber = flipBlockNumber + initializer.Config.EndingBlockNumber = flipBlockNumber - initializer := factories.LogNoteTransformer{ - Config: config, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel @@ -77,21 +92,17 @@ var _ = Describe("Deal transformer", func() { It("persists a flap deal log event", func() { flapBlockNumber := int64(9004628) - err := persistHeader(db, flapBlockNumber, blockChain) + header, err := persistHeader(db, flapBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := deal.DealConfig - config.StartingBlockNumber = flapBlockNumber - config.EndingBlockNumber = flapBlockNumber + initializer.Config.StartingBlockNumber = flapBlockNumber + initializer.Config.EndingBlockNumber = flapBlockNumber - initializer := factories.LogNoteTransformer{ - Config: config, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index 47e6c7ac..d68977b8 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -1,6 +1,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -15,8 +16,14 @@ import ( var _ = Describe("Dent transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + fetcher shared.Fetcher + transformer shared.Transformer + config shared.TransformerConfig + addresses []common.Address + topics []common.Hash + initializer factories.LogNoteTransformer ) BeforeEach(func() { @@ -26,25 +33,31 @@ var _ = Describe("Dent transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = dent.DentConfig + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} + fetcher = shared.NewFetcher(blockChain) + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &dent.DentConverter{}, + Repository: &dent.DentRepository{}, + } }) It("persists a flop dent log event", func() { blockNumber := int64(8955613) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := dent.DentConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - initializer := factories.LogNoteTransformer{ - Config: config, - Converter: &dent.DentConverter{}, - Repository: &dent.DentRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []dent.DentModel diff --git a/pkg/transformers/integration_tests/drip_drip.go b/pkg/transformers/integration_tests/drip_drip.go index 6203b2dc..783616f0 100644 --- a/pkg/transformers/integration_tests/drip_drip.go +++ b/pkg/transformers/integration_tests/drip_drip.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,32 +28,43 @@ import ( ) var _ = Describe("DripDrip Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) It("transforms DripDrip log events", func() { blockNumber := int64(8934775) config := drip_drip.DripDripConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, Converter: &drip_drip.DripDripConverter{}, Repository: &drip_drip.DripDripRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) + transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute() + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []drip_drip.DripDripModel diff --git a/pkg/transformers/integration_tests/drip_file_vow.go b/pkg/transformers/integration_tests/drip_file_vow.go index 08630370..3f2f065b 100644 --- a/pkg/transformers/integration_tests/drip_file_vow.go +++ b/pkg/transformers/integration_tests/drip_file_vow.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,31 +28,44 @@ import ( ) var _ = Describe("Drip File Vow LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("transforms DripFileVow log events", func() { blockNumber := int64(8762197) config := vow.DripFileVowConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vow.DripFileVowConverter{}, Repository: &vow.DripFileVowRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vow.DripFileVowModel diff --git a/pkg/transformers/integration_tests/flap_kick.go b/pkg/transformers/integration_tests/flap_kick.go index 6b317f6e..f7ddee53 100644 --- a/pkg/transformers/integration_tests/flap_kick.go +++ b/pkg/transformers/integration_tests/flap_kick.go @@ -15,6 +15,10 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/test_config" "time" . "github.com/onsi/ginkgo" @@ -23,35 +27,46 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("FlapKick Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a FlapKick event from Kovan chain", func() { blockNumber := int64(9002933) config := flap_kick.FlapKickConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) - Expect(err).NotTo(HaveOccurred()) - - initializer := factories.Transformer{ + transformer := factories.Transformer{ Config: config, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + }.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flap_kick.FlapKickModel diff --git a/pkg/transformers/integration_tests/flip_kick.go b/pkg/transformers/integration_tests/flip_kick.go index 17076b47..598542d4 100644 --- a/pkg/transformers/integration_tests/flip_kick.go +++ b/pkg/transformers/integration_tests/flip_kick.go @@ -65,21 +65,26 @@ var _ = Describe("FlipKick Transformer", func() { Expect(err).NotTo(HaveOccurred()) blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ + transformer := factories.Transformer{ Config: config, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + }.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flip_kick.FlipKickModel diff --git a/pkg/transformers/integration_tests/flop_kick.go b/pkg/transformers/integration_tests/flop_kick.go index 93690d43..6ecab800 100644 --- a/pkg/transformers/integration_tests/flop_kick.go +++ b/pkg/transformers/integration_tests/flop_kick.go @@ -35,8 +35,13 @@ import ( var _ = Describe("FlopKick Transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + initializer factories.Transformer + fetcher shared.LogFetcher + addresses []common.Address + topics []common.Hash ) BeforeEach(func() { @@ -46,25 +51,33 @@ var _ = Describe("FlopKick Transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = flop_kick.Config + + initializer = factories.Transformer{ + Config: config, + Converter: &flop_kick.FlopKickConverter{}, + Repository: &flop_kick.FlopKickRepository{}, + } + + fetcher = shared.NewFetcher(blockChain) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} }) It("fetches and transforms a FlopKick event from Kovan chain", func() { blockNumber := int64(8672119) - config := flop_kick.Config - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ - Config: config, - Converter: &flop_kick.FlopKickConverter{}, - Repository: &flop_kick.FlopKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flop_kick.Model @@ -82,21 +95,17 @@ var _ = Describe("FlopKick Transformer", func() { It("fetches and transforms another FlopKick event from Kovan chain", func() { blockNumber := int64(8955611) - config := flop_kick.Config - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ - Config: config, - Converter: &flop_kick.FlopKickConverter{}, - Repository: &flop_kick.FlopKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flop_kick.Model diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index 00f31df4..b80e47c9 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -33,8 +33,11 @@ import ( var _ = Describe("Frob Transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + fetcher shared.Fetcher + config shared.TransformerConfig + initializer factories.Transformer ) BeforeEach(func() { @@ -44,25 +47,32 @@ var _ = Describe("Frob Transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + fetcher = shared.NewFetcher(blockChain) + config = frob.FrobConfig + initializer = factories.Transformer{ + Config: config, + Converter: &frob.FrobConverter{}, + Repository: &frob.FrobRepository{}, + } }) It("fetches and transforms a Frob event from Kovan chain", func() { blockNumber := int64(8935258) - config := frob.FrobConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ - Config: config, - Converter: &frob.FrobConverter{}, - Repository: &frob.FrobRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []frob.FrobModel diff --git a/pkg/transformers/integration_tests/helpers.go b/pkg/transformers/integration_tests/helpers.go index 19078a14..e44b0cce 100644 --- a/pkg/transformers/integration_tests/helpers.go +++ b/pkg/transformers/integration_tests/helpers.go @@ -42,12 +42,13 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor return blockChain, nil } -func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) error { +// Persist the header for a given block to postgres. Returns the header if successful. +func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) (core.Header, error) { header, err := blockChain.GetHeaderByNumber(blockNumber) if err != nil { - return err + return core.Header{}, err } headerRepository := repositories.NewHeaderRepository(db) _, err = headerRepository.CreateOrUpdateHeader(header) - return err + return header, err } diff --git a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go index f3eaff08..895487f2 100644 --- a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go +++ b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,31 +28,44 @@ import ( ) var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() { blockNumber := int64(8535578) config := debt_ceiling.DebtCeilingFileConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []debt_ceiling.PitFileDebtCeilingModel diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index 7a255141..81a57763 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" @@ -39,17 +40,24 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []ilk.PitFileIlkModel diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index fa8a13e2..a3ce2313 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,8 +30,11 @@ import ( var _ = Describe("Price feeds transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + fetcher shared.Fetcher + initializer factories.LogNoteTransformer ) BeforeEach(func() { @@ -40,30 +44,36 @@ var _ = Describe("Price feeds transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + config = price_feeds.PriceFeedConfig + fetcher = shared.NewFetcher(blockChain) + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &price_feeds.PriceFeedConverter{}, + Repository: &price_feeds.PriceFeedRepository{}, + } }) It("persists a ETH/USD price feed event", func() { blockNumber := int64(8763054) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := price_feeds.PriceFeedConfig - config.ContractAddresses = []string{constants.PipContractAddress} - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.ContractAddresses = []string{constants.PipContractAddress} + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - transformerInitializer := factories.LogNoteTransformer{ - Config: config, - Converter: &price_feeds.PriceFeedConverter{}, - Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(initializer.Config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel - err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("207.314891143000011198")) Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) @@ -71,26 +81,24 @@ var _ = Describe("Price feeds transformer", func() { It("persists a MKR/USD price feed event", func() { blockNumber := int64(8763059) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := price_feeds.PriceFeedConfig - config.ContractAddresses = []string{constants.PepContractAddress} - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.ContractAddresses = []string{constants.PepContractAddress} + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - transformerInitializer := factories.LogNoteTransformer{ - Config: config, - Converter: &price_feeds.PriceFeedConverter{}, - Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(initializer.Config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel - err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("391.803979212000001553")) Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) @@ -98,26 +106,24 @@ var _ = Describe("Price feeds transformer", func() { It("persists a REP/USD price feed event", func() { blockNumber := int64(8763062) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := price_feeds.PriceFeedConfig - config.ContractAddresses = []string{constants.RepContractAddress} - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.ContractAddresses = []string{constants.RepContractAddress} + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - transformerInitializer := factories.LogNoteTransformer{ - Config: config, - Converter: &price_feeds.PriceFeedConverter{}, - Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel - err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("12.816928482699999847")) Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index 5fe2084e..e2d7c3b6 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -28,8 +29,13 @@ import ( var _ = Describe("Tend LogNoteTransformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + fetcher shared.Fetcher + initializer factories.LogNoteTransformer + addresses []common.Address + topics []common.Hash ) BeforeEach(func() { @@ -39,25 +45,31 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + fetcher = shared.NewFetcher(blockChain) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} + + initializer = factories.LogNoteTransformer{ + Config: tend.TendConfig, + Converter: &tend.TendConverter{}, + Repository: &tend.TendRepository{}, + } }) It("fetches and transforms a Flip Tend event from Kovan chain", func() { blockNumber := int64(8935601) - config := tend.TendConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &tend.TendConverter{}, - Repository: &tend.TendRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel @@ -80,21 +92,17 @@ var _ = Describe("Tend LogNoteTransformer", func() { It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() { blockNumber := int64(8935731) - config := tend.TendConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &tend.TendConverter{}, - Repository: &tend.TendRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel @@ -117,21 +125,17 @@ var _ = Describe("Tend LogNoteTransformer", func() { It("fetches and transforms a Flap Tend event from the Kovan chain", func() { blockNumber := int64(9003177) - config := tend.TendConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &tend.TendConverter{}, - Repository: &tend.TendRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel diff --git a/pkg/transformers/integration_tests/vat_flux.go b/pkg/transformers/integration_tests/vat_flux.go index 901e294d..04ecb129 100644 --- a/pkg/transformers/integration_tests/vat_flux.go +++ b/pkg/transformers/integration_tests/vat_flux.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -39,17 +40,24 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_flux.VatFluxModel diff --git a/pkg/transformers/integration_tests/vat_fold.go b/pkg/transformers/integration_tests/vat_fold.go index 6331a89c..7e5a270a 100644 --- a/pkg/transformers/integration_tests/vat_fold.go +++ b/pkg/transformers/integration_tests/vat_fold.go @@ -18,39 +18,53 @@ import ( "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" - "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("VatFold Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("transforms VatFold log events", func() { blockNumber := int64(9051149) config := vat_fold.VatFoldConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_fold.VatFoldModel diff --git a/pkg/transformers/integration_tests/vat_grab.go b/pkg/transformers/integration_tests/vat_grab.go index f33f0918..f43346f2 100644 --- a/pkg/transformers/integration_tests/vat_grab.go +++ b/pkg/transformers/integration_tests/vat_grab.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" "math/big" . "github.com/onsi/ginkgo" @@ -41,17 +42,23 @@ var _ = Describe("Vat Grab Transformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) transformer := factories.LogNoteTransformer{ Config: config, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer(db, blockChain) + }.NewLogNoteTransformer(db) - err = transformer.Execute() + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_grab.VatGrabModel diff --git a/pkg/transformers/integration_tests/vat_heal.go b/pkg/transformers/integration_tests/vat_heal.go index 61c8ec0d..e787b642 100644 --- a/pkg/transformers/integration_tests/vat_heal.go +++ b/pkg/transformers/integration_tests/vat_heal.go @@ -40,17 +40,23 @@ var _ = Describe("VatHeal Transformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) transformer := factories.LogNoteTransformer{ Config: config, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer(db, blockChain) + }.NewLogNoteTransformer(db) - err = transformer.Execute() + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_heal.VatHealModel diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index 010f7429..e31c3aa4 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" @@ -39,17 +40,23 @@ var _ = Describe("VatInit LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_init.VatInitModel diff --git a/pkg/transformers/integration_tests/vat_move.go b/pkg/transformers/integration_tests/vat_move.go index c977e5cc..3ed85452 100644 --- a/pkg/transformers/integration_tests/vat_move.go +++ b/pkg/transformers/integration_tests/vat_move.go @@ -40,17 +40,23 @@ var _ = Describe("VatMove LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_move.VatMoveModel diff --git a/pkg/transformers/integration_tests/vat_slip.go b/pkg/transformers/integration_tests/vat_slip.go index 0b82914a..de937cca 100644 --- a/pkg/transformers/integration_tests/vat_slip.go +++ b/pkg/transformers/integration_tests/vat_slip.go @@ -1,6 +1,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -18,31 +19,38 @@ var _ = Describe("Vat slip transformer", func() { blockChain core.BlockChain ) - It("persists vat slip event", func() { + BeforeEach(func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + }) + It("persists vat slip event", func() { blockNumber := int64(8953655) config := vat_slip.VatSlipConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_slip.VatSlipConverter{}, Repository: &vat_slip.VatSlipRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) + }.NewLogNoteTransformer(db) - err = transformer.Execute() + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var headerID int64 diff --git a/pkg/transformers/integration_tests/vat_tune.go b/pkg/transformers/integration_tests/vat_tune.go index 59001f80..6a794867 100644 --- a/pkg/transformers/integration_tests/vat_tune.go +++ b/pkg/transformers/integration_tests/vat_tune.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" "math/big" . "github.com/onsi/ginkgo" @@ -41,17 +42,23 @@ var _ = Describe("VatTune LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_tune.VatTuneModel diff --git a/pkg/transformers/integration_tests/vow_flog.go b/pkg/transformers/integration_tests/vow_flog.go index 548086e0..494b4326 100644 --- a/pkg/transformers/integration_tests/vow_flog.go +++ b/pkg/transformers/integration_tests/vow_flog.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,32 +28,43 @@ import ( ) var _ = Describe("VowFlog LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("transforms VowFlog log events", func() { blockNumber := int64(8946819) config := vow_flog.VowFlogConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) - Expect(1).To(Equal(1)) - initializer := factories.LogNoteTransformer{ + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vow_flog.VowFlogConverter{}, Repository: &vow_flog.VowFlogRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vow_flog.VowFlogModel diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index 259a7f84..6caaac9f 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -27,7 +27,7 @@ type LogChunker struct { // Initialises a chunker by creating efficient lookup maps func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker { addressToNames := map[string][]string{} - nameToTopic0 := map[string]common.Hash{} + nameToTopic0 := map[string]common.Hash{} for _, config := range transformerConfigs { for _, address := range config.ContractAddresses { diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go index e07f939b..b5a290aa 100644 --- a/pkg/transformers/shared/repository_utility_test.go +++ b/pkg/transformers/shared/repository_utility_test.go @@ -87,4 +87,4 @@ func getExpectedColumnNames() []string { "vow_flog_checked", "flap_kick_checked", } -} \ No newline at end of file +} diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index a5ff5835..2673a6f3 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -47,3 +47,10 @@ func HexToString(byteString string) string { value := common.HexToHash(byteString) return value.Big().String() } + +func HexStringsToAddresses(strings []string) (addresses []common.Address) { + for _, hexString := range strings { + addresses = append(addresses, common.HexToAddress(hexString)) + } + return +} diff --git a/pkg/transformers/test_data/mocks/repository.go b/pkg/transformers/test_data/mocks/repository.go index 1cf5bf0a..0a998401 100644 --- a/pkg/transformers/test_data/mocks/repository.go +++ b/pkg/transformers/test_data/mocks/repository.go @@ -3,7 +3,6 @@ package mocks import ( . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -12,8 +11,6 @@ type MockRepository struct { markHeaderCheckedError error MarkHeaderCheckedPassedHeaderIDs []int64 CreatedHeaderIds []int64 - missingHeaders []core.Header - missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 @@ -36,24 +33,10 @@ func (repository *MockRepository) MarkHeaderChecked(headerID int64) error { return repository.markHeaderCheckedError } -func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - repository.PassedStartingBlockNumber = startingBlockNumber - repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersError -} - func (repository *MockRepository) SetDB(db *postgres.DB) { repository.SetDbCalled = true } -func (repository *MockRepository) SetMissingHeadersError(e error) { - repository.missingHeadersError = e -} - -func (repository *MockRepository) SetMissingHeaders(headers []core.Header) { - repository.missingHeaders = headers -} - func (repository *MockRepository) SetMarkHeaderCheckedError(e error) { repository.markHeaderCheckedError = e } diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index ae036206..cb95908b 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -79,7 +79,7 @@ var ( Repository: &flop_kick.FlopKickRepository{}, } - customEventTransformers = []factories.Transformer { + customEventTransformers = []factories.Transformer{ BiteTransformer, FlapKickTransformer, FlipKickTransformer, @@ -228,7 +228,7 @@ var ( Repository: &vat_flux.VatFluxRepository{}, } - logNoteTransformers = []factories.LogNoteTransformer { + logNoteTransformers = []factories.LogNoteTransformer{ CatFileChopLumpTransformer, CatFileFlipTransformer, CatFilePitVowTransformer, From 637623b752bcba068f450333227b8ca6f09ee180 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 15:40:45 +0100 Subject: [PATCH 11/33] Fix test suites! --- libraries/shared/watcher.go | 8 ++++++-- libraries/shared/watcher_test.go | 9 ++++++--- pkg/transformers/factories/log_note_transformer.go | 1 + pkg/transformers/factories/transformer.go | 1 + pkg/transformers/test_data/mocks/log_fetcher.go | 6 +++--- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 89ee5a42..79834d57 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -2,6 +2,7 @@ package shared import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers" @@ -18,7 +19,7 @@ type Watcher struct { Topics []common.Hash } -func NewWatcher(db postgres.DB, bc core.BlockChain) Watcher { +func NewWatcher(db postgres.DB, bc core.BlockChain, fetcher shared.LogFetcher) Watcher { transformerConfigs := transformers.TransformerConfigs() var contractAddresses []common.Address var topic0s []common.Hash @@ -31,7 +32,6 @@ func NewWatcher(db postgres.DB, bc core.BlockChain) Watcher { } chunker := shared.NewLogChunker(transformerConfigs) - fetcher := shared.NewFetcher(bc) return Watcher{ DB: db, @@ -73,6 +73,10 @@ func (watcher *Watcher) Execute() error { for _, transformer := range watcher.Transformers { logChunk := chunkedLogs[transformer.Name()] err = transformer.Execute(logChunk, header) + if err != nil { + log.Error("%v transformer failed to execute: %v", transformer.Name(), err) + return err + } } } return err diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index e6781d04..2386375a 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -11,6 +11,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/test_config" ) @@ -45,7 +46,7 @@ var _ = Describe("Watcher", func() { // TODO Add tests for aggregate fetching // TODO Add tests for MissingHeaders - It("Adds transformers", func() { + It("adds transformers", func() { watcher := shared.Watcher{} watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) @@ -54,7 +55,7 @@ var _ = Describe("Watcher", func() { Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{})) }) - It("Adds transformers from multiple sources", func() { + It("adds transformers from multiple sources", func() { watcher := shared.Watcher{} watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) @@ -69,12 +70,14 @@ var _ = Describe("Watcher", func() { watcher shared.Watcher fakeTransformer *MockTransformer headerRepository repositories.HeaderRepository + mockFetcher shared2.LogFetcher ) BeforeEach(func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) - watcher = shared.NewWatcher(*db, nil) + mockFetcher = &mocks.MockLogFetcher{} + watcher = shared.NewWatcher(*db, nil, mockFetcher) headerRepository = repositories.NewHeaderRepository(db) _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index 849d9619..200f0bce 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -44,6 +44,7 @@ func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Head log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err } + return nil } models, err := transformer.Converter.ToModels(logs) diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index eae4fd7f..d6a4c385 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -44,6 +44,7 @@ func (transformer Transformer) Execute(logs []types.Log, header core.Header) err log.Printf("Error marking header as checked in %v: %v", transformerName, err) return err } + return nil } entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs) diff --git a/pkg/transformers/test_data/mocks/log_fetcher.go b/pkg/transformers/test_data/mocks/log_fetcher.go index 47ce03aa..1eeee33e 100644 --- a/pkg/transformers/test_data/mocks/log_fetcher.go +++ b/pkg/transformers/test_data/mocks/log_fetcher.go @@ -21,7 +21,7 @@ import ( ) type MockLogFetcher struct { - FetchedContractAddresses [][]string + FetchedContractAddresses [][]common.Address FetchedTopics [][]common.Hash FetchedBlocks []int64 fetcherError error @@ -29,9 +29,9 @@ type MockLogFetcher struct { SetBcCalled bool } -func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { +func (mlf *MockLogFetcher) FetchLogs(contractAddresses []common.Address, topics []common.Hash, header core.Header) ([]types.Log, error) { mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses) - mlf.FetchedTopics = topics + mlf.FetchedTopics = [][]common.Hash{topics} mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber) return mlf.FetchedLogs, mlf.fetcherError From 587d2219d86f3b9af6dae53fd08df9853890624b Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 16:50:13 +0100 Subject: [PATCH 12/33] Remove bc references and cleanup pointers --- cmd/backfillMakerLogs.go | 7 +++---- cmd/continuousLogSync.go | 6 ++++-- libraries/shared/watcher.go | 23 ++++++++++------------- libraries/shared/watcher_test.go | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 07a3d003..339b9252 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -17,6 +17,7 @@ package cmd import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -46,10 +47,8 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } - watcher := shared.Watcher{ - DB: *db, - Blockchain: blockChain, - } + fetcher := shared2.NewFetcher(blockChain) + watcher := shared.NewWatcher(db, fetcher) watcher.AddTransformers(transformers.TransformerInitializers()) watcher.Execute() diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 9f081ea3..786ea5ea 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -58,9 +58,11 @@ func syncMakerLogs() { log.Fatal("Failed to initialize database.") } + fetcher := shared2.NewFetcher(blockChain) + watcher := shared.Watcher{ - DB: *db, - Blockchain: blockChain, + DB: db, + Fetcher: fetcher, } transformerInititalizers := getTransformerInititalizers(transformerNames) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 79834d57..d521f597 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -3,7 +3,6 @@ package shared import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -11,15 +10,14 @@ import ( type Watcher struct { Transformers []shared.Transformer - DB postgres.DB - Blockchain core.BlockChain + DB *postgres.DB Fetcher shared.LogFetcher Chunker shared.LogChunker Addresses []common.Address Topics []common.Hash } -func NewWatcher(db postgres.DB, bc core.BlockChain, fetcher shared.LogFetcher) Watcher { +func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher) Watcher { transformerConfigs := transformers.TransformerConfigs() var contractAddresses []common.Address var topic0s []common.Hash @@ -34,31 +32,30 @@ func NewWatcher(db postgres.DB, bc core.BlockChain, fetcher shared.LogFetcher) W chunker := shared.NewLogChunker(transformerConfigs) return Watcher{ - DB: db, - Blockchain: bc, - Fetcher: fetcher, - Chunker: chunker, - Addresses: contractAddresses, - Topics: topic0s, + DB: db, + Fetcher: fetcher, + Chunker: chunker, + Addresses: contractAddresses, + Topics: topic0s, } } func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { for _, transformerInitializer := range us { - transformer := transformerInitializer(&watcher.DB) + transformer := transformerInitializer(watcher.DB) watcher.Transformers = append(watcher.Transformers, transformer) } } func (watcher *Watcher) Execute() error { - checkedColumnNames, err := shared.GetCheckedColumnNames(&watcher.DB) + checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB) if err != nil { return err } notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) // TODO Handle start and end numbers in transformers? - missingHeaders, err := shared.MissingHeaders(0, -1, &watcher.DB, notCheckedSQL) + missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) for _, header := range missingHeaders { // TODO Extend FetchLogs for doing several blocks at a time diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 2386375a..a3007a60 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -77,7 +77,7 @@ var _ = Describe("Watcher", func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) mockFetcher = &mocks.MockLogFetcher{} - watcher = shared.NewWatcher(*db, nil, mockFetcher) + watcher = shared.NewWatcher(db, mockFetcher) headerRepository = repositories.NewHeaderRepository(db) _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) From 3cb8fe60d41c9fb70f1493d4906ec41dd02eca41 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 16:58:24 +0100 Subject: [PATCH 13/33] Fix persistHeader not returning proper header ID --- pkg/transformers/integration_tests/helpers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/transformers/integration_tests/helpers.go b/pkg/transformers/integration_tests/helpers.go index e44b0cce..f6d00278 100644 --- a/pkg/transformers/integration_tests/helpers.go +++ b/pkg/transformers/integration_tests/helpers.go @@ -49,6 +49,7 @@ func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChai return core.Header{}, err } headerRepository := repositories.NewHeaderRepository(db) - _, err = headerRepository.CreateOrUpdateHeader(header) + id, err := headerRepository.CreateOrUpdateHeader(header) + header.Id = id return header, err } From bc9fd17793dc6536bea5a03d9c8c8203e38deb6b Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 17:12:51 +0100 Subject: [PATCH 14/33] Fix remaining integration tests --- pkg/transformers/integration_tests/dent.go | 1 + .../integration_tests/price_feeds.go | 31 ++++++++++++------- pkg/transformers/integration_tests/tend.go | 4 ++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index d68977b8..196ca82b 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -57,6 +57,7 @@ var _ = Describe("Dent transformer", func() { logs, err := fetcher.FetchLogs(addresses, topics, header) Expect(err).NotTo(HaveOccurred()) + transformer = initializer.NewLogNoteTransformer(db) err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index a3ce2313..3cf8cc47 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -35,6 +35,7 @@ var _ = Describe("Price feeds transformer", func() { config shared.TransformerConfig fetcher shared.Fetcher initializer factories.LogNoteTransformer + topics []common.Hash ) BeforeEach(func() { @@ -44,7 +45,10 @@ var _ = Describe("Price feeds transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + config = price_feeds.PriceFeedConfig + topics = []common.Hash{common.HexToHash(config.Topic)} + fetcher = shared.NewFetcher(blockChain) initializer = factories.LogNoteTransformer{ @@ -58,13 +62,14 @@ var _ = Describe("Price feeds transformer", func() { blockNumber := int64(8763054) header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer.Config.ContractAddresses = []string{constants.PipContractAddress} + addresses := []string{constants.PipContractAddress} + initializer.Config.ContractAddresses = addresses initializer.Config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber logs, err := fetcher.FetchLogs( - shared.HexStringsToAddresses(initializer.Config.ContractAddresses), - []common.Hash{common.HexToHash(config.Topic)}, + shared.HexStringsToAddresses(addresses), + topics, header) Expect(err).NotTo(HaveOccurred()) @@ -76,20 +81,21 @@ var _ = Describe("Price feeds transformer", func() { err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("207.314891143000011198")) - Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) It("persists a MKR/USD price feed event", func() { blockNumber := int64(8763059) header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer.Config.ContractAddresses = []string{constants.PepContractAddress} + addresses := []string{constants.PepContractAddress} + initializer.Config.ContractAddresses = addresses initializer.Config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber logs, err := fetcher.FetchLogs( - shared.HexStringsToAddresses(initializer.Config.ContractAddresses), - []common.Hash{common.HexToHash(config.Topic)}, + shared.HexStringsToAddresses(addresses), + topics, header) Expect(err).NotTo(HaveOccurred()) @@ -101,20 +107,21 @@ var _ = Describe("Price feeds transformer", func() { err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("391.803979212000001553")) - Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) It("persists a REP/USD price feed event", func() { blockNumber := int64(8763062) header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer.Config.ContractAddresses = []string{constants.RepContractAddress} + addresses := []string{constants.RepContractAddress} + initializer.Config.ContractAddresses = addresses initializer.Config.StartingBlockNumber = blockNumber initializer.Config.EndingBlockNumber = blockNumber logs, err := fetcher.FetchLogs( - shared.HexStringsToAddresses(config.ContractAddresses), - []common.Hash{common.HexToHash(config.Topic)}, + shared.HexStringsToAddresses(addresses), + topics, header) Expect(err).NotTo(HaveOccurred()) @@ -126,6 +133,6 @@ var _ = Describe("Price feeds transformer", func() { err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("12.816928482699999847")) - Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) }) diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index e2d7c3b6..726f2279 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -46,12 +46,14 @@ var _ = Describe("Tend LogNoteTransformer", func() { db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + config = tend.TendConfig + fetcher = shared.NewFetcher(blockChain) addresses = shared.HexStringsToAddresses(config.ContractAddresses) topics = []common.Hash{common.HexToHash(config.Topic)} initializer = factories.LogNoteTransformer{ - Config: tend.TendConfig, + Config: config, Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, } From 2d81861ae586a976f3347d4d851f0c0db2f9f774 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 21:11:25 +0100 Subject: [PATCH 15/33] Add chunker tests --- pkg/transformers/shared/log_chunker.go | 21 ++-- pkg/transformers/shared/log_chunker_test.go | 122 ++++++++++++++++++++ 2 files changed, 134 insertions(+), 9 deletions(-) create mode 100644 pkg/transformers/shared/log_chunker_test.go diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index 6caaac9f..b6feb866 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -19,9 +19,11 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) +// TODO Add unit tests for LogChunker + type LogChunker struct { - addressToNames map[string][]string - nameToTopic0 map[string]common.Hash + AddressToNames map[string][]string + NameToTopic0 map[string]common.Hash } // Initialises a chunker by creating efficient lookup maps @@ -37,21 +39,22 @@ func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker { } return LogChunker{ - addressToNames, - nameToTopic0, + AddressToNames: addressToNames, + NameToTopic0: nameToTopic0, } } -// Goes through an array of logs, associating relevant logs with transformers -func (chunker LogChunker) ChunkLogs(logs []types.Log) (chunks map[string][]types.Log) { +// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers +func (chunker LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { + chunks := map[string][]types.Log{} for _, log := range logs { // Topic0 is not unique to each transformer, also need to consider the contract address - relevantTransformers := chunker.addressToNames[log.Address.String()] + relevantTransformers := chunker.AddressToNames[log.Address.String()] for _, transformer := range relevantTransformers { - if chunker.nameToTopic0[transformer] == log.Topics[0] { + if chunker.NameToTopic0[transformer] == log.Topics[0] { chunks[transformer] = append(chunks[transformer], log) } } } - return + return chunks } diff --git a/pkg/transformers/shared/log_chunker_test.go b/pkg/transformers/shared/log_chunker_test.go new file mode 100644 index 00000000..96c1cf96 --- /dev/null +++ b/pkg/transformers/shared/log_chunker_test.go @@ -0,0 +1,122 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var _ = Describe("Log chunker", func() { + var ( + configs []shared.TransformerConfig + chunker shared.LogChunker + ) + + BeforeEach(func() { + configA := shared.TransformerConfig{ + TransformerName: "TransformerA", + ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"}, + Topic: "0xA", + } + configB := shared.TransformerConfig{ + TransformerName: "TransformerB", + ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"}, + Topic: "0xB", + } + + configC := shared.TransformerConfig{ + TransformerName: "TransformerC", + ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"}, + Topic: "0xC", + } + + configs = []shared.TransformerConfig{configA, configB, configC} + chunker = shared.NewLogChunker(configs) + }) + + Describe("initialisation", func() { + It("creates lookup maps correctly", func() { + Expect(chunker.AddressToNames).To(Equal(map[string][]string{ + "0x00000000000000000000000000000000000000A1": []string{"TransformerA"}, + "0x00000000000000000000000000000000000000A2": []string{"TransformerA", "TransformerC"}, + "0x00000000000000000000000000000000000000B1": []string{"TransformerB"}, + })) + + Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{ + "TransformerA": common.HexToHash("0xA"), + "TransformerB": common.HexToHash("0xB"), + "TransformerC": common.HexToHash("0xC"), + })) + }) + }) + + Describe("ChunkLogs", func() { + It("only associates logs with relevant topic0 and address to transformers", func() { + logs := []types.Log{log1, log2, log3, log4, log5} + chunks := chunker.ChunkLogs(logs) + + Expect(chunks["TransformerA"]).To(And(ContainElement(log1),ContainElement(log4))) + Expect(chunks["TransformerB"]).To(BeEmpty()) + Expect(chunks["TransformerC"]).To(ContainElement(log5)) + }) + }) +}) + +var ( + // Match TransformerA + log1 = types.Log{ + Address: common.HexToAddress("0xA1"), + Topics: []common.Hash{ + common.HexToHash("0xA"), + common.HexToHash("0xLogTopic1"), + }, + } + // Match TransformerA address, but not topic0 + log2 = types.Log{ + Address: common.HexToAddress("0xA1"), + Topics: []common.Hash{ + common.HexToHash("0xB"), + common.HexToHash("0xLogTopic2"), + }, + } + // Match TransformerA topic, but TransformerB address + log3 = types.Log{ + Address: common.HexToAddress("0xB1"), + Topics: []common.Hash{ + common.HexToHash("0xA"), + common.HexToHash("0xLogTopic3"), + }, + } + // Match TransformerA, with the other address + log4 = types.Log{ + Address: common.HexToAddress("0xA2"), + Topics: []common.Hash{ + common.HexToHash("0xA"), + common.HexToHash("0xLogTopic4"), + }, + } + // Match TransformerC, which shares address with TransformerA + log5 = types.Log{ + Address: common.HexToAddress("0xA2"), + Topics: []common.Hash{ + common.HexToHash("0xC"), + common.HexToHash("0xLogTopic5"), + }, + } +) \ No newline at end of file From e03ccb094f8a061901a4a4799f53e373b6374bbe Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 21:12:19 +0100 Subject: [PATCH 16/33] Add error propagation and test todos for watcher --- libraries/shared/watcher.go | 5 +++++ libraries/shared/watcher_test.go | 27 +++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index d521f597..744eb341 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -56,12 +56,17 @@ func (watcher *Watcher) Execute() error { // TODO Handle start and end numbers in transformers? missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) + if err != nil { + log.Error("Fetching of missing headers failed in watcher!") + return err + } for _, header := range missingHeaders { // TODO Extend FetchLogs for doing several blocks at a time logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header) if err != nil { // TODO Handle fetch error in watcher + log.Error("Error while fetching logs in watcher") return err } diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index a3007a60..6f6634f8 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -41,10 +41,9 @@ func fakeTransformerInitializer(db *postgres.DB) shared2.Transformer { } var _ = Describe("Watcher", func() { - // TODO Add test for watcher setting the BC - // TODO Add tests for log chunk delegation - // TODO Add tests for aggregate fetching - // TODO Add tests for MissingHeaders + It("initialises correctly", func() { + // TODO Test watcher initialisation + }) It("adds transformers", func() { watcher := shared.Watcher{} @@ -102,5 +101,25 @@ var _ = Describe("Watcher", func() { Expect(err).To(HaveOccurred()) Expect(fakeTransformer.executeWasCalled).To(BeFalse()) }) + + It("calls MissingHeaders", func() { + // TODO Tests for calling MissingHeaders + }) + + It("returns an error if missingHeaders returns an error", func() { + // TODO Test for propagating missingHeaders error + }) + + It("calls the log fetcher", func() { + // TODO Test for calling FetchLogs + }) + + It("returns an error if the log fetcher returns an error", func() { + // TODO Test for propagating log fetcher error + }) + + It("passes only relevant logs to each transformer", func() { + // TODO Test log delegation + }) }) }) From 83593cb5f23e60bfed848648e26f77173f356c3a Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 10 Dec 2018 21:12:55 +0100 Subject: [PATCH 17/33] Cleanup fetcher and add test todos --- pkg/transformers/shared/log_fetcher.go | 11 ----------- pkg/transformers/shared/log_fetcher_test.go | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index 47f6f77d..9d0e9415 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -22,25 +22,14 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" ) -// TODO Check if Fetcher can be simplified with aggregate logic - type LogFetcher interface { FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) } -type SettableLogFetcher interface { - LogFetcher - SetBC(bc core.BlockChain) -} - type Fetcher struct { blockChain core.BlockChain } -func (fetcher *Fetcher) SetBC(bc core.BlockChain) { - fetcher.blockChain = bc -} - func NewFetcher(blockchain core.BlockChain) Fetcher { return Fetcher{ blockChain: blockchain, diff --git a/pkg/transformers/shared/log_fetcher_test.go b/pkg/transformers/shared/log_fetcher_test.go index 4a0c55fa..73deb656 100644 --- a/pkg/transformers/shared/log_fetcher_test.go +++ b/pkg/transformers/shared/log_fetcher_test.go @@ -26,7 +26,14 @@ import ( ) var _ = Describe("Fetcher", func() { + Describe("Iinitialisation", func() { + It("creates correct lookup maps", func() { + + }) + }) + Describe("FetchLogs", func() { + // TODO Add tests for aggregate fetching It("fetches logs based on the given query", func() { blockChain := fakes.NewMockBlockChain() fetcher := shared.NewFetcher(blockChain) From d93817f346bc76960a8c95c1e76629503ce9ff36 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 11 Dec 2018 11:35:13 +0100 Subject: [PATCH 18/33] Interface-ify shared repository --- cmd/backfillMakerLogs.go | 3 +- libraries/shared/watcher.go | 33 ++++++++++++------- pkg/transformers/bite/repository.go | 4 +-- .../cat_file/chop_lump/repository.go | 4 +-- pkg/transformers/cat_file/flip/repository.go | 4 +-- .../cat_file/pit_vow/repository.go | 4 +-- pkg/transformers/deal/repository.go | 4 +-- pkg/transformers/dent/repository.go | 4 +-- pkg/transformers/drip_drip/repository.go | 4 +-- pkg/transformers/drip_file/ilk/repository.go | 4 +-- pkg/transformers/drip_file/repo/repository.go | 4 +-- pkg/transformers/drip_file/vow/repository.go | 4 +-- pkg/transformers/flap_kick/repository.go | 4 +-- pkg/transformers/flip_kick/repository.go | 4 +-- pkg/transformers/flop_kick/repository.go | 4 +-- pkg/transformers/frob/repository.go | 4 +-- .../pit_file/debt_ceiling/repository.go | 4 +-- pkg/transformers/pit_file/ilk/repository.go | 4 +-- pkg/transformers/price_feeds/repository.go | 4 +-- pkg/transformers/shared/repository.go | 12 ++++--- .../shared/repository_utility_test.go | 8 ++--- pkg/transformers/tend/repository.go | 4 +-- pkg/transformers/vat_flux/repository.go | 4 +-- pkg/transformers/vat_fold/repository.go | 4 +-- pkg/transformers/vat_grab/repository.go | 4 +-- pkg/transformers/vat_heal/repository.go | 4 +-- pkg/transformers/vat_init/repository.go | 4 +-- pkg/transformers/vat_move/repository.go | 4 +-- pkg/transformers/vat_slip/repository.go | 4 +-- pkg/transformers/vat_toll/repository.go | 4 +-- pkg/transformers/vat_tune/repository.go | 4 +-- pkg/transformers/vow_flog/repository.go | 4 +-- 32 files changed, 90 insertions(+), 78 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 339b9252..f8856825 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -47,8 +47,9 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } + repository := shared2.Repository{} fetcher := shared2.NewFetcher(blockChain) - watcher := shared.NewWatcher(db, fetcher) + watcher := shared.NewWatcher(db, fetcher, repository) watcher.AddTransformers(transformers.TransformerInitializers()) watcher.Execute() diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 744eb341..a9f94d91 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -3,11 +3,18 @@ package shared import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) +type WatcherRepository interface { + GetCheckedColumnNames(db *postgres.DB) ([]string, error) + CreateNotCheckedSQL(boolColumns []string) string + MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) +} + type Watcher struct { Transformers []shared.Transformer DB *postgres.DB @@ -15,9 +22,10 @@ type Watcher struct { Chunker shared.LogChunker Addresses []common.Address Topics []common.Hash + Repository WatcherRepository } -func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher) Watcher { +func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository) Watcher { transformerConfigs := transformers.TransformerConfigs() var contractAddresses []common.Address var topic0s []common.Hash @@ -32,11 +40,12 @@ func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher) Watcher { chunker := shared.NewLogChunker(transformerConfigs) return Watcher{ - DB: db, - Fetcher: fetcher, - Chunker: chunker, - Addresses: contractAddresses, - Topics: topic0s, + DB: db, + Fetcher: fetcher, + Chunker: chunker, + Addresses: contractAddresses, + Topics: topic0s, + Repository: repository, } } @@ -48,14 +57,14 @@ func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { } func (watcher *Watcher) Execute() error { - checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB) + checkedColumnNames, err := watcher.Repository.GetCheckedColumnNames(watcher.DB) if err != nil { return err } - notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) + notCheckedSQL := watcher.Repository.CreateNotCheckedSQL(checkedColumnNames) - // TODO Handle start and end numbers in transformers? - missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) + // TODO Handle start and end numbers in transformers + missingHeaders, err := watcher.Repository.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) if err != nil { log.Error("Fetching of missing headers failed in watcher!") return err @@ -66,7 +75,7 @@ func (watcher *Watcher) Execute() error { logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header) if err != nil { // TODO Handle fetch error in watcher - log.Error("Error while fetching logs in watcher") + log.Error("Error while fetching logs for header %v in watcher", header.Id) return err } @@ -76,7 +85,7 @@ func (watcher *Watcher) Execute() error { logChunk := chunkedLogs[transformer.Name()] err = transformer.Execute(logChunk, header) if err != nil { - log.Error("%v transformer failed to execute: %v", transformer.Name(), err) + log.Error("%v transformer failed to execute in watcher: %v", transformer.Name(), err) return err } } diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 1e3d860b..290deaee 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -52,7 +52,7 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.BiteChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.BiteChecked) if err != nil { tx.Rollback() return err @@ -62,5 +62,5 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er } func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) } diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index 9dff41f7..c63e83b6 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -49,7 +49,7 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileChopLumpChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileChopLumpChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte } func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) } func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/cat_file/flip/repository.go b/pkg/transformers/cat_file/flip/repository.go index a07ca207..624e37d3 100644 --- a/pkg/transformers/cat_file/flip/repository.go +++ b/pkg/transformers/cat_file/flip/repository.go @@ -48,7 +48,7 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileFlipChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileFlipChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac } func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) } func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/cat_file/pit_vow/repository.go b/pkg/transformers/cat_file/pit_vow/repository.go index 861dc2a8..8e2e3a29 100644 --- a/pkg/transformers/cat_file/pit_vow/repository.go +++ b/pkg/transformers/cat_file/pit_vow/repository.go @@ -48,7 +48,7 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFilePitVowChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFilePitVowChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf } func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) } func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go index 06f23230..825f00d6 100644 --- a/pkg/transformers/deal/repository.go +++ b/pkg/transformers/deal/repository.go @@ -49,7 +49,7 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DealChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DealChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er } func (repository DealRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) } func (repository *DealRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index 0ff5370b..80aee3e0 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -54,7 +54,7 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) if err != nil { tx.Rollback() return err @@ -63,7 +63,7 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er } func (repository DentRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) + return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) } func (repository *DentRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index edb1208a..c9fa1623 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -48,7 +48,7 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripDripChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripDripChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} } func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) } func (repository *DripDripRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 117a28fd..3740c548 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -50,7 +50,7 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileIlkChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileIlkChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac } func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) } func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index dc9589a7..23d08af2 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -50,7 +50,7 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileRepoChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileRepoChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa } func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) } func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index aa23bea8..86bdf8f2 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -49,7 +49,7 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileVowChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileVowChecked) if err != nil { tx.Rollback() return err @@ -59,7 +59,7 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac } func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) } func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/flap_kick/repository.go b/pkg/transformers/flap_kick/repository.go index f9f11c7a..950ebcb0 100644 --- a/pkg/transformers/flap_kick/repository.go +++ b/pkg/transformers/flap_kick/repository.go @@ -47,7 +47,7 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{ } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlapKickChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlapKickChecked) if err != nil { tx.Rollback() return err @@ -56,7 +56,7 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{ } func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) } func (repository *FlapKickRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/flip_kick/repository.go b/pkg/transformers/flip_kick/repository.go index db8a8d01..99c376be 100644 --- a/pkg/transformers/flip_kick/repository.go +++ b/pkg/transformers/flip_kick/repository.go @@ -47,7 +47,7 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{} return err } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlipKickChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlipKickChecked) if err != nil { tx.Rollback() return err @@ -56,7 +56,7 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{} } func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) + return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) } func (repository *FlipKickRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/flop_kick/repository.go b/pkg/transformers/flop_kick/repository.go index 7a302c75..1bab62c6 100644 --- a/pkg/transformers/flop_kick/repository.go +++ b/pkg/transformers/flop_kick/repository.go @@ -48,7 +48,7 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{} } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlopKickChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlopKickChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{} } func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) + return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) } func (repository *FlopKickRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index bf6d4d08..e2c26435 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -46,7 +46,7 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er return err } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FrobChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FrobChecked) if err != nil { tx.Rollback() return err @@ -55,7 +55,7 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er } func (repository FrobRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) } func (repository *FrobRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 12f69fb2..38b648ba 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -50,7 +50,7 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileDebtCeilingChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileDebtCeilingChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i } func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) } func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 9a187de7..3dae51c6 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -49,7 +49,7 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileIlkChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileIlkChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface } func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) } func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go index cbb9a012..af1caf19 100644 --- a/pkg/transformers/price_feeds/repository.go +++ b/pkg/transformers/price_feeds/repository.go @@ -45,7 +45,7 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{ } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.PriceFeedsChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.PriceFeedsChecked) if err != nil { tx.Rollback() return err @@ -54,7 +54,7 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{ } func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) } func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index 0380d50a..65b4f938 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -9,7 +9,9 @@ import ( "strings" ) -func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { +type Repository struct{} + +func (_ Repository) MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2) ON CONFLICT (header_id) DO @@ -17,7 +19,7 @@ func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn str return err } -func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error { +func (_ Repository) MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2) ON CONFLICT (header_id) DO @@ -26,7 +28,7 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo } // Treats a header as missing if it's not in the headers table, or not checked for some log type -func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { +func (_ Repository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { var result []core.Header var query string var err error @@ -51,7 +53,7 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D return result, err } -func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { +func (_ Repository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { // Query returns `[]driver.Value`, nullable polymorphic interface var queryResult []driver.Value columnNamesQuery := @@ -83,7 +85,7 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { // Defaults to FALSE when no columns are provided. // Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" // [] => "FALSE" -func CreateNotCheckedSQL(boolColumns []string) string { +func (_ Repository) CreateNotCheckedSQL(boolColumns []string) string { var result strings.Builder if len(boolColumns) == 0 { diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go index b5a290aa..b24cde03 100644 --- a/pkg/transformers/shared/repository_utility_test.go +++ b/pkg/transformers/shared/repository_utility_test.go @@ -27,7 +27,7 @@ var _ = Describe("Repository utilities", func() { db := test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) expectedColumnNames := getExpectedColumnNames() - actualColumnNames, err := shared.GetCheckedColumnNames(db) + actualColumnNames, err := shared.Repository{}.GetCheckedColumnNames(db) Expect(err).NotTo(HaveOccurred()) Expect(actualColumnNames).To(Equal(expectedColumnNames)) }) @@ -37,20 +37,20 @@ var _ = Describe("Repository utilities", func() { It("generates a correct SQL string for one column", func() { columns := []string{"columnA"} expected := "NOT (columnA)" - actual := shared.CreateNotCheckedSQL(columns) + actual := shared.Repository{}.CreateNotCheckedSQL(columns) Expect(actual).To(Equal(expected)) }) It("generates a correct SQL string for several columns", func() { columns := []string{"columnA", "columnB"} expected := "NOT (columnA AND columnB)" - actual := shared.CreateNotCheckedSQL(columns) + actual := shared.Repository{}.CreateNotCheckedSQL(columns) Expect(actual).To(Equal(expected)) }) It("defaults to FALSE when there are no columns", func() { expected := "FALSE" - actual := shared.CreateNotCheckedSQL([]string{}) + actual := shared.Repository{}.CreateNotCheckedSQL([]string{}) Expect(actual).To(Equal(expected)) }) }) diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 17620a00..bfabd248 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -55,7 +55,7 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.TendChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.TendChecked) if err != nil { tx.Rollback() return err @@ -64,7 +64,7 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er } func (repository TendRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) + return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) } func (repository *TendRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index 2febe527..78089701 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -47,7 +47,7 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFluxChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFluxChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{}) } func (repository VatFluxRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) + return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) } func (repository *VatFluxRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index 5d831bca..01ba28ba 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -49,7 +49,7 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFoldChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFoldChecked) if err != nil { tx.Rollback() return err @@ -59,7 +59,7 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) } func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) } func (repository *VatFoldRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index a8a274a4..14364b90 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -34,7 +34,7 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) return err } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatGrabChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatGrabChecked) if err != nil { tx.Rollback() return err @@ -43,7 +43,7 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) } func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) } func (repository *VatGrabRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_heal/repository.go b/pkg/transformers/vat_heal/repository.go index aed8fcd0..861025a6 100644 --- a/pkg/transformers/vat_heal/repository.go +++ b/pkg/transformers/vat_heal/repository.go @@ -52,7 +52,7 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatHealChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatHealChecked) if err != nil { tx.Rollback() return err @@ -61,5 +61,5 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) } func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error { - return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) + return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) } diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index 14b2d4c8..1bb56d12 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -49,7 +49,7 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatInitChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatInitChecked) if err != nil { tx.Rollback() return err @@ -59,7 +59,7 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) } func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) } func (repository *VatInitRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index 2548c2b4..913b587c 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -50,7 +50,7 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatMoveChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatMoveChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) } func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) } func (repository *VatMoveRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index 15f412ca..b84f248c 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -34,7 +34,7 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatSlipChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatSlipChecked) if err != nil { tx.Rollback() return err @@ -44,7 +44,7 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) } func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) } func (repository *VatSlipRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index 2c9be8a4..6c738e54 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -34,7 +34,7 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTollChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTollChecked) if err != nil { tx.Rollback() return err @@ -43,7 +43,7 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{}) } func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) } func (repository *VatTollRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index c9951ce6..79404cb0 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -34,7 +34,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTuneChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTuneChecked) if err != nil { tx.Rollback() return err @@ -43,7 +43,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) } func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) } func (repository *VatTuneRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vow_flog/repository.go b/pkg/transformers/vow_flog/repository.go index 48076802..994de3e2 100644 --- a/pkg/transformers/vow_flog/repository.go +++ b/pkg/transformers/vow_flog/repository.go @@ -50,7 +50,7 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{}) } } - err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VowFlogChecked) + err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VowFlogChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{}) } func (repository VowFlogRepository) MarkHeaderChecked(headerID int64) error { - return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) + return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) } func (repository *VowFlogRepository) SetDB(db *postgres.DB) { From 5d4239de05b6901b8b658c96337c33284d2a1648 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 11 Dec 2018 13:47:04 +0100 Subject: [PATCH 19/33] Finish watcher and watcher tests (+fmt) --- libraries/shared/watcher.go | 5 +- libraries/shared/watcher_test.go | 109 ++++++++++++++---- pkg/transformers/shared/log_chunker.go | 4 +- pkg/transformers/shared/log_chunker_test.go | 4 +- pkg/transformers/shared/log_fetcher_test.go | 7 -- .../test_data/mocks/log_fetcher.go | 2 + .../mocks/mock_watcher_repository.go | 53 +++++++++ 7 files changed, 149 insertions(+), 35 deletions(-) create mode 100644 pkg/transformers/test_data/mocks/mock_watcher_repository.go diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index a9f94d91..77f06112 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -5,7 +5,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) @@ -25,8 +24,8 @@ type Watcher struct { Repository WatcherRepository } -func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository) Watcher { - transformerConfigs := transformers.TransformerConfigs() +func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, + transformerConfigs []shared.TransformerConfig) Watcher { var contractAddresses []common.Address var topic0s []common.Hash diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 6f6634f8..0b32fcfe 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -2,6 +2,7 @@ package shared_test import ( "errors" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -10,6 +11,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/test_config" @@ -42,7 +44,28 @@ func fakeTransformerInitializer(db *postgres.DB) shared2.Transformer { var _ = Describe("Watcher", func() { It("initialises correctly", func() { - // TODO Test watcher initialisation + db := test_config.NewTestDB(core.Node{ID: "testNode"}) + fetcher := mocks.MockLogFetcher{} + repository := &mocks.MockWatcherRepository{} + configA := shared2.TransformerConfig{ + ContractAddresses: []string{"0xA"}, + Topic: "0xA", + } + configB := shared2.TransformerConfig{ + ContractAddresses: []string{"0xB"}, + Topic: "0xB", + } + configs := []shared2.TransformerConfig{configA, configB} + watcher := shared.NewWatcher(db, &fetcher, repository, configs) + + Expect(watcher.DB).To(Equal(db)) + Expect(watcher.Fetcher).NotTo(BeNil()) + Expect(watcher.Chunker).NotTo(BeNil()) + Expect(watcher.Repository).To(Equal(repository)) + Expect(watcher.Topics).To(And( + ContainElement(common.HexToHash("0xA")), ContainElement(common.HexToHash("0xB")))) + Expect(watcher.Addresses).To(And( + ContainElement(common.HexToAddress("0xA")), ContainElement(common.HexToAddress("0xB")))) }) It("adds transformers", func() { @@ -69,22 +92,26 @@ var _ = Describe("Watcher", func() { watcher shared.Watcher fakeTransformer *MockTransformer headerRepository repositories.HeaderRepository - mockFetcher shared2.LogFetcher + mockFetcher mocks.MockLogFetcher + repository mocks.MockWatcherRepository ) BeforeEach(func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) - mockFetcher = &mocks.MockLogFetcher{} - watcher = shared.NewWatcher(db, mockFetcher) + mockFetcher = mocks.MockLogFetcher{} headerRepository = repositories.NewHeaderRepository(db) _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) + + repository = mocks.MockWatcherRepository{} + watcher = shared.NewWatcher(db, &mockFetcher, &repository, transformers.TransformerConfigs()) }) It("executes each transformer", func() { fakeTransformer = &MockTransformer{} watcher.Transformers = []shared2.Transformer{fakeTransformer} + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) err := watcher.Execute() @@ -95,6 +122,7 @@ var _ = Describe("Watcher", func() { It("returns an error if transformer returns an error", func() { fakeTransformer = &MockTransformer{executeError: errors.New("Something bad happened")} watcher.Transformers = []shared2.Transformer{fakeTransformer} + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) err := watcher.Execute() @@ -102,24 +130,65 @@ var _ = Describe("Watcher", func() { Expect(fakeTransformer.executeWasCalled).To(BeFalse()) }) - It("calls MissingHeaders", func() { - // TODO Tests for calling MissingHeaders - }) - - It("returns an error if missingHeaders returns an error", func() { - // TODO Test for propagating missingHeaders error - }) - - It("calls the log fetcher", func() { - // TODO Test for calling FetchLogs - }) - - It("returns an error if the log fetcher returns an error", func() { - // TODO Test for propagating log fetcher error - }) - It("passes only relevant logs to each transformer", func() { // TODO Test log delegation }) + + Describe("uses the repository correctly:", func() { + + It("calls MissingHeaders", func() { + err := watcher.Execute() + Expect(err).To(Not(HaveOccurred())) + Expect(repository.MissingHeadersCalled).To(BeTrue()) + }) + + It("propagates MissingHeaders errors", func() { + missingHeadersError := errors.New("MissingHeadersError") + repository.MissingHeadersError = missingHeadersError + + err := watcher.Execute() + Expect(err).To(MatchError(missingHeadersError)) + }) + + It("calls CreateNotCheckedSQL", func() { + err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(repository.CreateNotCheckedSQLCalled).To(BeTrue()) + }) + + It("calls GetCheckedColumnNames", func() { + err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(repository.GetCheckedColumnNamesCalled).To(BeTrue()) + }) + + It("propagates GetCheckedColumnNames errors", func() { + getCheckedColumnNamesError := errors.New("GetCheckedColumnNamesError") + repository.GetCheckedColumnNamesError = getCheckedColumnNamesError + + err := watcher.Execute() + Expect(err).To(MatchError(getCheckedColumnNamesError)) + }) + }) + + Describe("uses the LogFetcher correctly:", func() { + BeforeEach(func() { + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + }) + + It("fetches logs", func() { + err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(mockFetcher.FetchLogsCalled).To(BeTrue()) + }) + + It("propagates log fetcher errors", func() { + fetcherError := errors.New("FetcherError") + mockFetcher.SetFetcherError(fetcherError) + + err := watcher.Execute() + Expect(err).To(MatchError(fetcherError)) + }) + }) }) }) diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index b6feb866..aede460a 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -19,8 +19,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// TODO Add unit tests for LogChunker - type LogChunker struct { AddressToNames map[string][]string NameToTopic0 map[string]common.Hash @@ -40,7 +38,7 @@ func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker { return LogChunker{ AddressToNames: addressToNames, - NameToTopic0: nameToTopic0, + NameToTopic0: nameToTopic0, } } diff --git a/pkg/transformers/shared/log_chunker_test.go b/pkg/transformers/shared/log_chunker_test.go index 96c1cf96..e657d1b4 100644 --- a/pkg/transformers/shared/log_chunker_test.go +++ b/pkg/transformers/shared/log_chunker_test.go @@ -71,7 +71,7 @@ var _ = Describe("Log chunker", func() { logs := []types.Log{log1, log2, log3, log4, log5} chunks := chunker.ChunkLogs(logs) - Expect(chunks["TransformerA"]).To(And(ContainElement(log1),ContainElement(log4))) + Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4))) Expect(chunks["TransformerB"]).To(BeEmpty()) Expect(chunks["TransformerC"]).To(ContainElement(log5)) }) @@ -119,4 +119,4 @@ var ( common.HexToHash("0xLogTopic5"), }, } -) \ No newline at end of file +) diff --git a/pkg/transformers/shared/log_fetcher_test.go b/pkg/transformers/shared/log_fetcher_test.go index 73deb656..4a0c55fa 100644 --- a/pkg/transformers/shared/log_fetcher_test.go +++ b/pkg/transformers/shared/log_fetcher_test.go @@ -26,14 +26,7 @@ import ( ) var _ = Describe("Fetcher", func() { - Describe("Iinitialisation", func() { - It("creates correct lookup maps", func() { - - }) - }) - Describe("FetchLogs", func() { - // TODO Add tests for aggregate fetching It("fetches logs based on the given query", func() { blockChain := fakes.NewMockBlockChain() fetcher := shared.NewFetcher(blockChain) diff --git a/pkg/transformers/test_data/mocks/log_fetcher.go b/pkg/transformers/test_data/mocks/log_fetcher.go index 1eeee33e..ff3db503 100644 --- a/pkg/transformers/test_data/mocks/log_fetcher.go +++ b/pkg/transformers/test_data/mocks/log_fetcher.go @@ -27,12 +27,14 @@ type MockLogFetcher struct { fetcherError error FetchedLogs []types.Log SetBcCalled bool + FetchLogsCalled bool } func (mlf *MockLogFetcher) FetchLogs(contractAddresses []common.Address, topics []common.Hash, header core.Header) ([]types.Log, error) { mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses) mlf.FetchedTopics = [][]common.Hash{topics} mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber) + mlf.FetchLogsCalled = true return mlf.FetchedLogs, mlf.fetcherError } diff --git a/pkg/transformers/test_data/mocks/mock_watcher_repository.go b/pkg/transformers/test_data/mocks/mock_watcher_repository.go new file mode 100644 index 00000000..323fdddf --- /dev/null +++ b/pkg/transformers/test_data/mocks/mock_watcher_repository.go @@ -0,0 +1,53 @@ +package mocks + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type MockWatcherRepository struct { + ReturnCheckedColumnNames []string + GetCheckedColumnNamesError error + GetCheckedColumnNamesCalled bool + + ReturnNotCheckedSQL string + CreateNotCheckedSQLCalled bool + + ReturnMissingHeaders []core.Header + MissingHeadersError error + MissingHeadersCalled bool +} + +func (repository *MockWatcherRepository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { + repository.GetCheckedColumnNamesCalled = true + if repository.GetCheckedColumnNamesError != nil { + return []string{}, repository.GetCheckedColumnNamesError + } + + return repository.ReturnCheckedColumnNames, nil +} + +func (repository *MockWatcherRepository) SetCheckedColumnNames(checkedColumnNames []string) { + repository.ReturnCheckedColumnNames = checkedColumnNames +} + +func (repository *MockWatcherRepository) CreateNotCheckedSQL(boolColumns []string) string { + repository.CreateNotCheckedSQLCalled = true + return repository.ReturnNotCheckedSQL +} + +func (repository *MockWatcherRepository) SetNotCheckedSQL(notCheckedSql string) { + repository.ReturnNotCheckedSQL = notCheckedSql +} + +func (repository *MockWatcherRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { + if repository.MissingHeadersError != nil { + return []core.Header{}, repository.MissingHeadersError + } + repository.MissingHeadersCalled = true + return repository.ReturnMissingHeaders, nil +} + +func (repository *MockWatcherRepository) SetMissingHeaders(headers []core.Header) { + repository.ReturnMissingHeaders = headers +} From 9000c6e9f9e50595ae8e90a9447655ad2018f737 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 11 Dec 2018 15:02:32 +0100 Subject: [PATCH 20/33] Add test for watcher log delegation --- libraries/shared/watcher_test.go | 41 ++++++++++++++++++++++---- pkg/transformers/shared/log_chunker.go | 2 ++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 0b32fcfe..d9f5976c 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -22,6 +22,7 @@ type MockTransformer struct { executeError error passedLogs []types.Log passedHeader core.Header + transformerName string } func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { @@ -35,7 +36,11 @@ func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { } func (mh *MockTransformer) Name() string { - return "MockTransformer" + return mh.transformerName +} + +func (mh *MockTransformer) SetTransformerName(name string) { + mh.transformerName = name } func fakeTransformerInitializer(db *postgres.DB) shared2.Transformer { @@ -49,11 +54,11 @@ var _ = Describe("Watcher", func() { repository := &mocks.MockWatcherRepository{} configA := shared2.TransformerConfig{ ContractAddresses: []string{"0xA"}, - Topic: "0xA", + Topic: "0xA", } configB := shared2.TransformerConfig{ ContractAddresses: []string{"0xB"}, - Topic: "0xB", + Topic: "0xB", } configs := []shared2.TransformerConfig{configA, configB} watcher := shared.NewWatcher(db, &fetcher, repository, configs) @@ -93,7 +98,7 @@ var _ = Describe("Watcher", func() { fakeTransformer *MockTransformer headerRepository repositories.HeaderRepository mockFetcher mocks.MockLogFetcher - repository mocks.MockWatcherRepository + repository mocks.MockWatcherRepository ) BeforeEach(func() { @@ -131,7 +136,33 @@ var _ = Describe("Watcher", func() { }) It("passes only relevant logs to each transformer", func() { - // TODO Test log delegation + transformerA := &MockTransformer{} + transformerA.SetTransformerName("transformerA") + transformerB := &MockTransformer{} + transformerB.SetTransformerName("transformerB") + + configA := shared2.TransformerConfig{TransformerName: "transformerA", + ContractAddresses: []string{"0x000000000000000000000000000000000000000A"}, + Topic: "0xA"} + configB := shared2.TransformerConfig{TransformerName: "transformerB", + ContractAddresses: []string{"0x000000000000000000000000000000000000000b"}, + Topic: "0xB"} + configs := []shared2.TransformerConfig{configA, configB} + + logA := types.Log{Address: common.HexToAddress("0xA"), + Topics: []common.Hash{common.HexToHash("0xA")}} + logB := types.Log{Address: common.HexToAddress("0xB"), + Topics: []common.Hash{common.HexToHash("0xB")}} + mockFetcher.SetFetchedLogs([]types.Log{logA, logB}) + + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + watcher = shared.NewWatcher(db, &mockFetcher, &repository, configs) + watcher.Transformers = []shared2.Transformer{transformerA, transformerB} + + err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(transformerA.passedLogs).To(Equal([]types.Log{logA})) + Expect(transformerB.passedLogs).To(Equal([]types.Log{logB})) }) Describe("uses the repository correctly:", func() { diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index aede460a..a7ebcc17 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -48,6 +48,8 @@ func (chunker LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { for _, log := range logs { // Topic0 is not unique to each transformer, also need to consider the contract address relevantTransformers := chunker.AddressToNames[log.Address.String()] + + // TODO What should happen if log can't be assigned? for _, transformer := range relevantTransformers { if chunker.NameToTopic0[transformer] == log.Topics[0] { chunks[transformer] = append(chunks[transformer], log) From 4e089c363e096c5a1c7de88f14437a706f913aa1 Mon Sep 17 00:00:00 2001 From: Edvard Date: Tue, 11 Dec 2018 15:19:27 +0100 Subject: [PATCH 21/33] Fixes after merging staging --- cmd/backfillMakerLogs.go | 2 +- libraries/shared/watcher.go | 2 +- libraries/shared/watcher_test.go | 4 +- .../integration_tests/pit_file_ilk.go | 70 +++++++++---------- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index f8856825..0c424d4a 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -49,7 +49,7 @@ func backfillMakerLogs() { repository := shared2.Repository{} fetcher := shared2.NewFetcher(blockChain) - watcher := shared.NewWatcher(db, fetcher, repository) + watcher := shared.NewWatcher(db, fetcher, repository, transformers.TransformerConfigs()) watcher.AddTransformers(transformers.TransformerInitializers()) watcher.Execute() diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 77f06112..207b0cc0 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -25,7 +25,7 @@ type Watcher struct { } func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, - transformerConfigs []shared.TransformerConfig) Watcher { + transformerConfigs []shared.TransformerConfig) Watcher { var contractAddresses []common.Address var topic0s []common.Hash diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index d9f5976c..cf11f1d4 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -143,10 +143,10 @@ var _ = Describe("Watcher", func() { configA := shared2.TransformerConfig{TransformerName: "transformerA", ContractAddresses: []string{"0x000000000000000000000000000000000000000A"}, - Topic: "0xA"} + Topic: "0xA"} configB := shared2.TransformerConfig{TransformerName: "transformerB", ContractAddresses: []string{"0x000000000000000000000000000000000000000b"}, - Topic: "0xB"} + Topic: "0xB"} configs := []shared2.TransformerConfig{configA, configB} logA := types.Log{Address: common.HexToAddress("0xA"), diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index a0be6b58..ab713a94 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -18,6 +18,8 @@ import ( "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" @@ -26,37 +28,46 @@ import ( ) var _ = Describe("PitFileIlk LogNoteTransformer", func() { - It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() { - blockNumber := int64(9103223) - config := ilk.IlkFileConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + var ( + db *postgres.DB + blockChain core.BlockChain + initializer factories.LogNoteTransformer + addresses []common.Address + topics []common.Hash + ) + BeforeEach(func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockChain.Node()) + db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + config := ilk.IlkFileConfig - header, err := persistHeader(db, blockNumber, blockChain) - Expect(err).NotTo(HaveOccurred()) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} - initializer := factories.LogNoteTransformer{ + initializer = factories.LogNoteTransformer{ Config: config, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, } - transformer := initializer.NewLogNoteTransformer(db) + }) - fetcher := shared.NewFetcher(blockChain) - logs, err := fetcher.FetchLogs( - shared.HexStringsToAddresses(config.ContractAddresses), - []common.Hash{common.HexToHash(config.Topic)}, - header) + It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() { + blockNumber := int64(9103223) + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) @@ -72,29 +83,18 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() { blockNumber := int64(8762253) - config := ilk.IlkFileConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs(addresses, topics, header) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &ilk.PitFileIlkConverter{}, - Repository: &ilk.PitFileIlkRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []ilk.PitFileIlkModel From a86a2ec5c6cc483a33f9b2415def3eb071a48571 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 12 Dec 2018 12:13:23 +0100 Subject: [PATCH 22/33] Improve checked header column detection --- pkg/transformers/shared/repository.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index 65b4f938..129da6e9 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -60,8 +60,7 @@ func (_ Repository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { `SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'checked_headers' - AND column_name != 'id' - AND column_name != 'header_id';` + AND column_name ~ '_checked';` err := db.Select(&queryResult, columnNamesQuery) if err != nil { From dc43547612dd6cd20e75fe09cec879bcff8fc414 Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 12 Dec 2018 15:41:29 +0100 Subject: [PATCH 23/33] Improve process of adding transformers --- cmd/backfillMakerLogs.go | 7 +- cmd/continuousLogSync.go | 130 ++++++++++++++++--------- libraries/shared/watcher.go | 47 +++++---- pkg/transformers/shared/log_chunker.go | 30 +++--- pkg/transformers/shared/log_fetcher.go | 4 +- 5 files changed, 134 insertions(+), 84 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 0c424d4a..56986c85 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -47,11 +47,12 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } - repository := shared2.Repository{} + repository := &shared2.Repository{} fetcher := shared2.NewFetcher(blockChain) - watcher := shared.NewWatcher(db, fetcher, repository, transformers.TransformerConfigs()) + chunker := shared2.NewLogChunker() + watcher := shared.NewWatcher(db, fetcher, repository, chunker) - watcher.AddTransformers(transformers.TransformerInitializers()) + watcher.AddTransformers(transformers.TransformerInitializers(), transformers.TransformerConfigs()) watcher.Execute() } diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 786ea5ea..9a70cbe4 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -59,71 +59,109 @@ func syncMakerLogs() { } fetcher := shared2.NewFetcher(blockChain) + repository := &shared2.Repository{} + chunker := shared2.NewLogChunker() - watcher := shared.Watcher{ - DB: db, - Fetcher: fetcher, - } + initializers, configs := getTransformerSubset(transformerNames) + chunker.AddConfigs(configs) - transformerInititalizers := getTransformerInititalizers(transformerNames) - watcher.AddTransformers(transformerInititalizers) + watcher := shared.NewWatcher(db, fetcher, repository, chunker) + watcher.AddTransformers(initializers, configs) for range ticker.C { watcher.Execute() } } -func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer { - transformerInitializerMap := buildTransformerInitializerMap() - var transformerInitializers []shared2.TransformerInitializer +func getTransformerSubset(transformerNames []string) ([]shared2.TransformerInitializer, []shared2.TransformerConfig) { + var initializers []shared2.TransformerInitializer + var configs []shared2.TransformerConfig if transformerNames[0] == "all" { - for _, v := range transformerInitializerMap { - transformerInitializers = append(transformerInitializers, v) - } + initializers = transformers.TransformerInitializers() + configs = transformers.TransformerConfigs() } else { + initializerMap := buildTransformerInitializerMap() + configMap := buildTransformerConfigMap() + for _, transformerName := range transformerNames { - initializer := transformerInitializerMap[transformerName] - transformerInitializers = append(transformerInitializers, initializer) + initializers = append(initializers, initializerMap[transformerName]) + configs = append(configs, configMap[transformerName]) } } - - return transformerInitializers + return initializers, configs } func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { - transformerInitializerMap := make(map[string]shared2.TransformerInitializer) + initializerMap := make(map[string]shared2.TransformerInitializer) - transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer - transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer - transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer - transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer - transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer - transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer - transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer - transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer + initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer + initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer + initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer + initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer + initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer + initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer + initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer + initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer + initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer + initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer + initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer + initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer + initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer + initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer + initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer + initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer + initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer + initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer + initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer + initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer + initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer + initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer + initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer + initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer - return transformerInitializerMap + return initializerMap +} + +func buildTransformerConfigMap() map[string]shared2.TransformerConfig { + configMap := make(map[string]shared2.TransformerConfig) + + configMap[constants.BiteLabel] = transformers.BiteTransformer.Config + configMap[constants.BiteLabel] = transformers.BiteTransformer.Config + configMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.Config + configMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.Config + configMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.Config + configMap[constants.DealLabel] = transformers.DealTransformer.Config + configMap[constants.DentLabel] = transformers.DentTransformer.Config + configMap[constants.DripDripLabel] = transformers.DripDripTransformer.Config + configMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.Config + configMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.Config + configMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.Config + configMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.Config + configMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.Config + configMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.Config + configMap[constants.FrobLabel] = transformers.FrobTransformer.Config + configMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.Config + configMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.Config + configMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.Config + configMap[constants.TendLabel] = transformers.TendTransformer.Config + configMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.Config + configMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.Config + configMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.Config + configMap[constants.VatHealLabel] = transformers.VatHealTransformer.Config + configMap[constants.VatInitLabel] = transformers.VatInitTransformer.Config + configMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.Config + configMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.Config + configMap[constants.VatTollLabel] = transformers.VatTollTransformer.Config + configMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.Config + configMap[constants.VowFlogLabel] = transformers.FlogTransformer.Config + + return configMap } func init() { diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 207b0cc0..e5c96851 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -18,41 +18,46 @@ type Watcher struct { Transformers []shared.Transformer DB *postgres.DB Fetcher shared.LogFetcher - Chunker shared.LogChunker + Chunker shared.Chunker Addresses []common.Address Topics []common.Hash Repository WatcherRepository } -func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, - transformerConfigs []shared.TransformerConfig) Watcher { +func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, chunker shared.Chunker) Watcher { + return Watcher{ + DB: db, + Fetcher: fetcher, + Chunker: chunker, + Repository: repository, + } +} + +// Adds transformers to the watcher, each needs an initializer and the associated config. +// This also changes the configuration of the chunker, so that it will consider the new transformers. +func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer, configs []shared.TransformerConfig) { + if len(initializers) != len(configs) { + panic("Mismatch in number of transformers initializers and configs!") + } + + for _, initializer := range initializers { + transformer := initializer(watcher.DB) + watcher.Transformers = append(watcher.Transformers, transformer) + } + var contractAddresses []common.Address var topic0s []common.Hash - for _, config := range transformerConfigs { + for _, config := range configs { for _, address := range config.ContractAddresses { contractAddresses = append(contractAddresses, common.HexToAddress(address)) } topic0s = append(topic0s, common.HexToHash(config.Topic)) } - chunker := shared.NewLogChunker(transformerConfigs) - - return Watcher{ - DB: db, - Fetcher: fetcher, - Chunker: chunker, - Addresses: contractAddresses, - Topics: topic0s, - Repository: repository, - } -} - -func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { - for _, transformerInitializer := range us { - transformer := transformerInitializer(watcher.DB) - watcher.Transformers = append(watcher.Transformers, transformer) - } + watcher.Addresses = append(watcher.Addresses, contractAddresses...) + watcher.Topics = append(watcher.Topics, topic0s...) + watcher.Chunker.AddConfigs(configs) } func (watcher *Watcher) Execute() error { diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index a7ebcc17..dea03a5b 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -19,31 +19,37 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) +type Chunker interface { + AddConfigs(transformerConfigs []TransformerConfig) + ChunkLogs(logs []types.Log) map[string][]types.Log +} + type LogChunker struct { AddressToNames map[string][]string NameToTopic0 map[string]common.Hash } -// Initialises a chunker by creating efficient lookup maps -func NewLogChunker(transformerConfigs []TransformerConfig) LogChunker { - addressToNames := map[string][]string{} - nameToTopic0 := map[string]common.Hash{} +// Returns a new log chunker with initialised maps. +// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer. +func NewLogChunker() *LogChunker { + return &LogChunker{ + AddressToNames: map[string][]string{}, + NameToTopic0: map[string]common.Hash{}, + } +} +// Configures the chunker by adding more addreses and topics to consider. +func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) { for _, config := range transformerConfigs { for _, address := range config.ContractAddresses { - addressToNames[address] = append(addressToNames[address], config.TransformerName) - nameToTopic0[config.TransformerName] = common.HexToHash(config.Topic) + chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName) + chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic) } } - - return LogChunker{ - AddressToNames: addressToNames, - NameToTopic0: nameToTopic0, - } } // Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers -func (chunker LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { +func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { chunks := map[string][]types.Log{} for _, log := range logs { // Topic0 is not unique to each transformer, also need to consider the contract address diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index 9d0e9415..11081a24 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -30,8 +30,8 @@ type Fetcher struct { blockChain core.BlockChain } -func NewFetcher(blockchain core.BlockChain) Fetcher { - return Fetcher{ +func NewFetcher(blockchain core.BlockChain) *Fetcher { + return &Fetcher{ blockChain: blockchain, } } From 882a6dd7a22dbfe95a5b3086e1af573dc58714ca Mon Sep 17 00:00:00 2001 From: Edvard Date: Wed, 12 Dec 2018 15:41:44 +0100 Subject: [PATCH 24/33] Update tests with new adding of transformers --- libraries/shared/watcher_test.go | 56 ++++++++++--------- .../integration_tests/cat_file.go | 2 +- pkg/transformers/integration_tests/deal.go | 2 +- pkg/transformers/integration_tests/dent.go | 2 +- pkg/transformers/integration_tests/frob.go | 2 +- .../integration_tests/price_feeds.go | 2 +- pkg/transformers/integration_tests/tend.go | 2 +- pkg/transformers/shared/log_chunker_test.go | 19 ++++++- 8 files changed, 54 insertions(+), 33 deletions(-) diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index cf11f1d4..7b91ece9 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -11,7 +11,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/transformers" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" "github.com/vulcanize/vulcanizedb/test_config" @@ -47,48 +46,51 @@ func fakeTransformerInitializer(db *postgres.DB) shared2.Transformer { return &MockTransformer{} } +var fakeTransformerConfig = []shared2.TransformerConfig{{ + TransformerName: "FakeTransformer", + ContractAddresses: []string{"FakeAddress"}, + Topic: "FakeTopic", +}} + var _ = Describe("Watcher", func() { It("initialises correctly", func() { db := test_config.NewTestDB(core.Node{ID: "testNode"}) - fetcher := mocks.MockLogFetcher{} + fetcher := &mocks.MockLogFetcher{} repository := &mocks.MockWatcherRepository{} - configA := shared2.TransformerConfig{ - ContractAddresses: []string{"0xA"}, - Topic: "0xA", - } - configB := shared2.TransformerConfig{ - ContractAddresses: []string{"0xB"}, - Topic: "0xB", - } - configs := []shared2.TransformerConfig{configA, configB} - watcher := shared.NewWatcher(db, &fetcher, repository, configs) + chunker := shared2.NewLogChunker() + + watcher := shared.NewWatcher(db, fetcher, repository, chunker) Expect(watcher.DB).To(Equal(db)) - Expect(watcher.Fetcher).NotTo(BeNil()) - Expect(watcher.Chunker).NotTo(BeNil()) + Expect(watcher.Fetcher).To(Equal(fetcher)) + Expect(watcher.Chunker).To(Equal(chunker)) Expect(watcher.Repository).To(Equal(repository)) - Expect(watcher.Topics).To(And( - ContainElement(common.HexToHash("0xA")), ContainElement(common.HexToHash("0xB")))) - Expect(watcher.Addresses).To(And( - ContainElement(common.HexToAddress("0xA")), ContainElement(common.HexToAddress("0xB")))) }) It("adds transformers", func() { - watcher := shared.Watcher{} + chunker := shared2.NewLogChunker() + watcher := shared.NewWatcher(nil, nil, nil, chunker) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}, fakeTransformerConfig) Expect(len(watcher.Transformers)).To(Equal(1)) Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{})) + Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")})) + Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")})) }) It("adds transformers from multiple sources", func() { - watcher := shared.Watcher{} + chunker := shared2.NewLogChunker() + watcher := shared.NewWatcher(nil, nil, nil, chunker) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}, fakeTransformerConfig) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}, fakeTransformerConfig) Expect(len(watcher.Transformers)).To(Equal(2)) + Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"), + common.HexToHash("FakeTopic")})) + Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"), + common.HexToAddress("FakeAddress")})) }) Describe("with missing headers", func() { @@ -99,6 +101,7 @@ var _ = Describe("Watcher", func() { headerRepository repositories.HeaderRepository mockFetcher mocks.MockLogFetcher repository mocks.MockWatcherRepository + chunker *shared2.LogChunker ) BeforeEach(func() { @@ -108,9 +111,10 @@ var _ = Describe("Watcher", func() { headerRepository = repositories.NewHeaderRepository(db) _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) + chunker = shared2.NewLogChunker() repository = mocks.MockWatcherRepository{} - watcher = shared.NewWatcher(db, &mockFetcher, &repository, transformers.TransformerConfigs()) + watcher = shared.NewWatcher(db, &mockFetcher, &repository, chunker) }) It("executes each transformer", func() { @@ -155,8 +159,10 @@ var _ = Describe("Watcher", func() { Topics: []common.Hash{common.HexToHash("0xB")}} mockFetcher.SetFetchedLogs([]types.Log{logA, logB}) + chunker.AddConfigs(configs) + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - watcher = shared.NewWatcher(db, &mockFetcher, &repository, configs) + watcher = shared.NewWatcher(db, &mockFetcher, &repository, chunker) watcher.Transformers = []shared2.Transformer{transformerA, transformerB} err := watcher.Execute() diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 059e37af..06679e83 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -40,7 +40,7 @@ var _ = Describe("Cat File transformer", func() { rpcClient client.RpcClient err error ethClient *ethclient.Client - fetcher shared.Fetcher + fetcher *shared.Fetcher ) BeforeEach(func() { diff --git a/pkg/transformers/integration_tests/deal.go b/pkg/transformers/integration_tests/deal.go index 990b0d76..b010cced 100644 --- a/pkg/transformers/integration_tests/deal.go +++ b/pkg/transformers/integration_tests/deal.go @@ -34,7 +34,7 @@ var _ = Describe("Deal transformer", func() { blockChain core.BlockChain config shared.TransformerConfig initializer factories.LogNoteTransformer - fetcher shared.Fetcher + fetcher *shared.Fetcher addresses []common.Address topics []common.Hash ) diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index 196ca82b..d1c8050c 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -18,7 +18,7 @@ var _ = Describe("Dent transformer", func() { var ( db *postgres.DB blockChain core.BlockChain - fetcher shared.Fetcher + fetcher *shared.Fetcher transformer shared.Transformer config shared.TransformerConfig addresses []common.Address diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index b80e47c9..50cde0a0 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -35,7 +35,7 @@ var _ = Describe("Frob Transformer", func() { var ( db *postgres.DB blockChain core.BlockChain - fetcher shared.Fetcher + fetcher *shared.Fetcher config shared.TransformerConfig initializer factories.Transformer ) diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index 3cf8cc47..fac1f71d 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -33,7 +33,7 @@ var _ = Describe("Price feeds transformer", func() { db *postgres.DB blockChain core.BlockChain config shared.TransformerConfig - fetcher shared.Fetcher + fetcher *shared.Fetcher initializer factories.LogNoteTransformer topics []common.Hash ) diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index 726f2279..ae6938ba 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -32,7 +32,7 @@ var _ = Describe("Tend LogNoteTransformer", func() { db *postgres.DB blockChain core.BlockChain config shared.TransformerConfig - fetcher shared.Fetcher + fetcher *shared.Fetcher initializer factories.LogNoteTransformer addresses []common.Address topics []common.Hash diff --git a/pkg/transformers/shared/log_chunker_test.go b/pkg/transformers/shared/log_chunker_test.go index e657d1b4..7dfd3d69 100644 --- a/pkg/transformers/shared/log_chunker_test.go +++ b/pkg/transformers/shared/log_chunker_test.go @@ -25,7 +25,7 @@ import ( var _ = Describe("Log chunker", func() { var ( configs []shared.TransformerConfig - chunker shared.LogChunker + chunker *shared.LogChunker ) BeforeEach(func() { @@ -47,7 +47,8 @@ var _ = Describe("Log chunker", func() { } configs = []shared.TransformerConfig{configA, configB, configC} - chunker = shared.NewLogChunker(configs) + chunker = shared.NewLogChunker() + chunker.AddConfigs(configs) }) Describe("initialisation", func() { @@ -66,6 +67,20 @@ var _ = Describe("Log chunker", func() { }) }) + Describe("AddConfigs", func() { + It("can add more configs later", func() { + configD := shared.TransformerConfig{ + TransformerName: "TransformerD", + ContractAddresses: []string{"0x000000000000000000000000000000000000000D"}, + Topic: "0xD", + } + chunker.AddConfigs([]shared.TransformerConfig{configD}) + + Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"})) + Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD"))) + }) + }) + Describe("ChunkLogs", func() { It("only associates logs with relevant topic0 and address to transformers", func() { logs := []types.Log{log1, log2, log3, log4, log5} From d30fcfed807fdeea2eaf36c287ace82b74bf4af9 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 13 Dec 2018 12:39:57 +0100 Subject: [PATCH 25/33] Use new config getter on shared.Transformer <3 --- cmd/backfillMakerLogs.go | 10 +-- cmd/continuousLogSync.go | 58 +++-------------- libraries/shared/watcher.go | 32 +++++----- libraries/shared/watcher_test.go | 62 +++++++++---------- .../factories/log_note_transformer.go | 6 +- pkg/transformers/factories/transformer.go | 6 +- pkg/transformers/shared/transformer.go | 2 +- pkg/transformers/transformers.go | 12 ---- 8 files changed, 73 insertions(+), 115 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 56986c85..4f19038e 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -49,11 +49,13 @@ func backfillMakerLogs() { repository := &shared2.Repository{} fetcher := shared2.NewFetcher(blockChain) - chunker := shared2.NewLogChunker() - watcher := shared.NewWatcher(db, fetcher, repository, chunker) + watcher := shared.NewWatcher(db, fetcher, repository) - watcher.AddTransformers(transformers.TransformerInitializers(), transformers.TransformerConfigs()) - watcher.Execute() + watcher.AddTransformers(transformers.TransformerInitializers()) + err = watcher.Execute() + if err != nil { + // TODO Handle watcher error in backfillMakerLogs + } } func init() { diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 9a70cbe4..9f3e2fec 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -60,36 +60,32 @@ func syncMakerLogs() { fetcher := shared2.NewFetcher(blockChain) repository := &shared2.Repository{} - chunker := shared2.NewLogChunker() - initializers, configs := getTransformerSubset(transformerNames) - chunker.AddConfigs(configs) + initializers := getTransformerInitializers(transformerNames) - watcher := shared.NewWatcher(db, fetcher, repository, chunker) - watcher.AddTransformers(initializers, configs) + watcher := shared.NewWatcher(db, fetcher, repository) + watcher.AddTransformers(initializers) for range ticker.C { - watcher.Execute() + err = watcher.Execute() + if err != nil { + // TODO Handle watcher errors in ContinuousLogSync + } } } -func getTransformerSubset(transformerNames []string) ([]shared2.TransformerInitializer, []shared2.TransformerConfig) { +func getTransformerInitializers(transformerNames []string) []shared2.TransformerInitializer { var initializers []shared2.TransformerInitializer - var configs []shared2.TransformerConfig if transformerNames[0] == "all" { initializers = transformers.TransformerInitializers() - configs = transformers.TransformerConfigs() } else { initializerMap := buildTransformerInitializerMap() - configMap := buildTransformerConfigMap() - for _, transformerName := range transformerNames { initializers = append(initializers, initializerMap[transformerName]) - configs = append(configs, configMap[transformerName]) } } - return initializers, configs + return initializers } func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { @@ -128,42 +124,6 @@ func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer return initializerMap } -func buildTransformerConfigMap() map[string]shared2.TransformerConfig { - configMap := make(map[string]shared2.TransformerConfig) - - configMap[constants.BiteLabel] = transformers.BiteTransformer.Config - configMap[constants.BiteLabel] = transformers.BiteTransformer.Config - configMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.Config - configMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.Config - configMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.Config - configMap[constants.DealLabel] = transformers.DealTransformer.Config - configMap[constants.DentLabel] = transformers.DentTransformer.Config - configMap[constants.DripDripLabel] = transformers.DripDripTransformer.Config - configMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.Config - configMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.Config - configMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.Config - configMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.Config - configMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.Config - configMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.Config - configMap[constants.FrobLabel] = transformers.FrobTransformer.Config - configMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.Config - configMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.Config - configMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.Config - configMap[constants.TendLabel] = transformers.TendTransformer.Config - configMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.Config - configMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.Config - configMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.Config - configMap[constants.VatHealLabel] = transformers.VatHealTransformer.Config - configMap[constants.VatInitLabel] = transformers.VatInitTransformer.Config - configMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.Config - configMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.Config - configMap[constants.VatTollLabel] = transformers.VatTollTransformer.Config - configMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.Config - configMap[constants.VowFlogLabel] = transformers.FlogTransformer.Config - - return configMap -} - func init() { rootCmd.AddCommand(continuousLogSyncCmd) continuousLogSyncCmd.Flags().StringSliceVar(&transformerNames, "transformers", []string{"all"}, "transformer names to be run during this command") diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index e5c96851..7b68752c 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -24,7 +24,8 @@ type Watcher struct { Repository WatcherRepository } -func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository, chunker shared.Chunker) Watcher { +func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository) Watcher { + chunker := shared.NewLogChunker() return Watcher{ DB: db, Fetcher: fetcher, @@ -33,25 +34,21 @@ func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRe } } -// Adds transformers to the watcher, each needs an initializer and the associated config. -// This also changes the configuration of the chunker, so that it will consider the new transformers. -func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer, configs []shared.TransformerConfig) { - if len(initializers) != len(configs) { - panic("Mismatch in number of transformers initializers and configs!") - } +// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers. +func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) { + var contractAddresses []common.Address + var topic0s []common.Hash + var configs []shared.TransformerConfig for _, initializer := range initializers { transformer := initializer(watcher.DB) watcher.Transformers = append(watcher.Transformers, transformer) - } - var contractAddresses []common.Address - var topic0s []common.Hash + config := transformer.GetConfig() + configs = append(configs, config) - for _, config := range configs { - for _, address := range config.ContractAddresses { - contractAddresses = append(contractAddresses, common.HexToAddress(address)) - } + addresses := shared.HexStringsToAddresses(config.ContractAddresses) + contractAddresses = append(contractAddresses, addresses...) topic0s = append(topic0s, common.HexToHash(config.Topic)) } @@ -85,11 +82,14 @@ func (watcher *Watcher) Execute() error { chunkedLogs := watcher.Chunker.ChunkLogs(logs) + // Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync, + // not all logs we're interested in might have been fetched. for _, transformer := range watcher.Transformers { - logChunk := chunkedLogs[transformer.Name()] + transformerName := transformer.GetConfig().TransformerName + logChunk := chunkedLogs[transformerName] err = transformer.Execute(logChunk, header) if err != nil { - log.Error("%v transformer failed to execute in watcher: %v", transformer.Name(), err) + log.Error("%v transformer failed to execute in watcher: %v", transformerName, err) return err } } diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 7b91ece9..5fc3c0c5 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -21,7 +21,7 @@ type MockTransformer struct { executeError error passedLogs []types.Log passedHeader core.Header - transformerName string + config shared2.TransformerConfig } func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { @@ -34,57 +34,60 @@ func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { return nil } -func (mh *MockTransformer) Name() string { - return mh.transformerName +func (mh *MockTransformer) GetConfig() shared2.TransformerConfig { + return mh.config } -func (mh *MockTransformer) SetTransformerName(name string) { - mh.transformerName = name +func (mh *MockTransformer) SetTransformerConfig(config shared2.TransformerConfig) { + mh.config = config } -func fakeTransformerInitializer(db *postgres.DB) shared2.Transformer { - return &MockTransformer{} +func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared2.Transformer { + return mh } -var fakeTransformerConfig = []shared2.TransformerConfig{{ +var fakeTransformerConfig = shared2.TransformerConfig{ TransformerName: "FakeTransformer", ContractAddresses: []string{"FakeAddress"}, Topic: "FakeTopic", -}} +} var _ = Describe("Watcher", func() { It("initialises correctly", func() { db := test_config.NewTestDB(core.Node{ID: "testNode"}) fetcher := &mocks.MockLogFetcher{} repository := &mocks.MockWatcherRepository{} - chunker := shared2.NewLogChunker() - watcher := shared.NewWatcher(db, fetcher, repository, chunker) + watcher := shared.NewWatcher(db, fetcher, repository) Expect(watcher.DB).To(Equal(db)) Expect(watcher.Fetcher).To(Equal(fetcher)) - Expect(watcher.Chunker).To(Equal(chunker)) + Expect(watcher.Chunker).NotTo(BeNil()) Expect(watcher.Repository).To(Equal(repository)) }) It("adds transformers", func() { - chunker := shared2.NewLogChunker() - watcher := shared.NewWatcher(nil, nil, nil, chunker) - - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}, fakeTransformerConfig) + watcher := shared.NewWatcher(nil, nil, nil) + fakeTransformer := &MockTransformer{} + fakeTransformer.SetTransformerConfig(fakeTransformerConfig) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) Expect(len(watcher.Transformers)).To(Equal(1)) - Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{})) + Expect(watcher.Transformers).To(ConsistOf(fakeTransformer)) Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")})) Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")})) }) It("adds transformers from multiple sources", func() { - chunker := shared2.NewLogChunker() - watcher := shared.NewWatcher(nil, nil, nil, chunker) + watcher := shared.NewWatcher(nil, nil, nil) + fakeTransformer1 := &MockTransformer{} + fakeTransformer1.SetTransformerConfig(fakeTransformerConfig) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}, fakeTransformerConfig) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}, fakeTransformerConfig) + fakeTransformer2 := &MockTransformer{} + fakeTransformer2.SetTransformerConfig(fakeTransformerConfig) + + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer}) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer}) Expect(len(watcher.Transformers)).To(Equal(2)) Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"), @@ -101,7 +104,6 @@ var _ = Describe("Watcher", func() { headerRepository repositories.HeaderRepository mockFetcher mocks.MockLogFetcher repository mocks.MockWatcherRepository - chunker *shared2.LogChunker ) BeforeEach(func() { @@ -111,10 +113,9 @@ var _ = Describe("Watcher", func() { headerRepository = repositories.NewHeaderRepository(db) _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) - chunker = shared2.NewLogChunker() repository = mocks.MockWatcherRepository{} - watcher = shared.NewWatcher(db, &mockFetcher, &repository, chunker) + watcher = shared.NewWatcher(db, &mockFetcher, &repository) }) It("executes each transformer", func() { @@ -141,9 +142,7 @@ var _ = Describe("Watcher", func() { It("passes only relevant logs to each transformer", func() { transformerA := &MockTransformer{} - transformerA.SetTransformerName("transformerA") transformerB := &MockTransformer{} - transformerB.SetTransformerName("transformerB") configA := shared2.TransformerConfig{TransformerName: "transformerA", ContractAddresses: []string{"0x000000000000000000000000000000000000000A"}, @@ -151,7 +150,9 @@ var _ = Describe("Watcher", func() { configB := shared2.TransformerConfig{TransformerName: "transformerB", ContractAddresses: []string{"0x000000000000000000000000000000000000000b"}, Topic: "0xB"} - configs := []shared2.TransformerConfig{configA, configB} + + transformerA.SetTransformerConfig(configA) + transformerB.SetTransformerConfig(configB) logA := types.Log{Address: common.HexToAddress("0xA"), Topics: []common.Hash{common.HexToHash("0xA")}} @@ -159,11 +160,10 @@ var _ = Describe("Watcher", func() { Topics: []common.Hash{common.HexToHash("0xB")}} mockFetcher.SetFetchedLogs([]types.Log{logA, logB}) - chunker.AddConfigs(configs) - repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - watcher = shared.NewWatcher(db, &mockFetcher, &repository, chunker) - watcher.Transformers = []shared2.Transformer{transformerA, transformerB} + watcher = shared.NewWatcher(db, &mockFetcher, &repository) + watcher.AddTransformers([]shared2.TransformerInitializer{ + transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer}) err := watcher.Execute() Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index 200f0bce..6ae63f4b 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -61,6 +61,10 @@ func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Head return nil } -func (transformer LogNoteTransformer) Name() string { +func (transformer LogNoteTransformer) GetName() string { return transformer.Config.TransformerName } + +func (transformer LogNoteTransformer) GetConfig() shared.TransformerConfig { + return transformer.Config +} diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index d6a4c385..5b164814 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -68,6 +68,10 @@ func (transformer Transformer) Execute(logs []types.Log, header core.Header) err return nil } -func (transformer Transformer) Name() string { +func (transformer Transformer) GetName() string { return transformer.Config.TransformerName } + +func (transformer Transformer) GetConfig() shared.TransformerConfig { + return transformer.Config +} diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 2673a6f3..0cc510d4 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -24,7 +24,7 @@ import ( type Transformer interface { Execute(logs []types.Log, header core.Header) error - Name() string + GetConfig() TransformerConfig } type TransformerInitializer func(db *postgres.DB) Transformer diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index cb95908b..6b0b7938 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -267,15 +267,3 @@ func TransformerInitializers() (initializers []shared.TransformerInitializer) { } return } - -// `TransformerConfigs` returns the config structs for all available transformers -func TransformerConfigs() (allConfigs []shared.TransformerConfig) { - for _, transformer := range logNoteTransformers { - allConfigs = append(allConfigs, transformer.Config) - } - - for _, transformer := range logNoteTransformers { - allConfigs = append(allConfigs, transformer.Config) - } - return -} From e5db740ae42eed9e56cf210e4f4afbcfe9b6016e Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 13 Dec 2018 13:03:51 +0100 Subject: [PATCH 26/33] Move MockTransformer to test_data --- libraries/shared/watcher_test.go | 66 +++++-------------- .../test_data/mocks/transformer.go | 44 +++++++++++++ 2 files changed, 59 insertions(+), 51 deletions(-) create mode 100644 pkg/transformers/test_data/mocks/transformer.go diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index 5fc3c0c5..c5ea6836 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -16,42 +16,6 @@ import ( "github.com/vulcanize/vulcanizedb/test_config" ) -type MockTransformer struct { - executeWasCalled bool - executeError error - passedLogs []types.Log - passedHeader core.Header - config shared2.TransformerConfig -} - -func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { - if mh.executeError != nil { - return mh.executeError - } - mh.executeWasCalled = true - mh.passedLogs = logs - mh.passedHeader = header - return nil -} - -func (mh *MockTransformer) GetConfig() shared2.TransformerConfig { - return mh.config -} - -func (mh *MockTransformer) SetTransformerConfig(config shared2.TransformerConfig) { - mh.config = config -} - -func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared2.Transformer { - return mh -} - -var fakeTransformerConfig = shared2.TransformerConfig{ - TransformerName: "FakeTransformer", - ContractAddresses: []string{"FakeAddress"}, - Topic: "FakeTopic", -} - var _ = Describe("Watcher", func() { It("initialises correctly", func() { db := test_config.NewTestDB(core.Node{ID: "testNode"}) @@ -68,8 +32,8 @@ var _ = Describe("Watcher", func() { It("adds transformers", func() { watcher := shared.NewWatcher(nil, nil, nil) - fakeTransformer := &MockTransformer{} - fakeTransformer.SetTransformerConfig(fakeTransformerConfig) + fakeTransformer := &mocks.MockTransformer{} + fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) Expect(len(watcher.Transformers)).To(Equal(1)) @@ -80,11 +44,11 @@ var _ = Describe("Watcher", func() { It("adds transformers from multiple sources", func() { watcher := shared.NewWatcher(nil, nil, nil) - fakeTransformer1 := &MockTransformer{} - fakeTransformer1.SetTransformerConfig(fakeTransformerConfig) + fakeTransformer1 := &mocks.MockTransformer{} + fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig) - fakeTransformer2 := &MockTransformer{} - fakeTransformer2.SetTransformerConfig(fakeTransformerConfig) + fakeTransformer2 := &mocks.MockTransformer{} + fakeTransformer2.SetTransformerConfig(mocks.FakeTransformerConfig) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer}) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer}) @@ -100,7 +64,7 @@ var _ = Describe("Watcher", func() { var ( db *postgres.DB watcher shared.Watcher - fakeTransformer *MockTransformer + fakeTransformer *mocks.MockTransformer headerRepository repositories.HeaderRepository mockFetcher mocks.MockLogFetcher repository mocks.MockWatcherRepository @@ -119,30 +83,30 @@ var _ = Describe("Watcher", func() { }) It("executes each transformer", func() { - fakeTransformer = &MockTransformer{} + fakeTransformer = &mocks.MockTransformer{} watcher.Transformers = []shared2.Transformer{fakeTransformer} repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) err := watcher.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(fakeTransformer.executeWasCalled).To(BeTrue()) + Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue()) }) It("returns an error if transformer returns an error", func() { - fakeTransformer = &MockTransformer{executeError: errors.New("Something bad happened")} + fakeTransformer = &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")} watcher.Transformers = []shared2.Transformer{fakeTransformer} repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) err := watcher.Execute() Expect(err).To(HaveOccurred()) - Expect(fakeTransformer.executeWasCalled).To(BeFalse()) + Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse()) }) It("passes only relevant logs to each transformer", func() { - transformerA := &MockTransformer{} - transformerB := &MockTransformer{} + transformerA := &mocks.MockTransformer{} + transformerB := &mocks.MockTransformer{} configA := shared2.TransformerConfig{TransformerName: "transformerA", ContractAddresses: []string{"0x000000000000000000000000000000000000000A"}, @@ -167,8 +131,8 @@ var _ = Describe("Watcher", func() { err := watcher.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(transformerA.passedLogs).To(Equal([]types.Log{logA})) - Expect(transformerB.passedLogs).To(Equal([]types.Log{logB})) + Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA})) + Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB})) }) Describe("uses the repository correctly:", func() { diff --git a/pkg/transformers/test_data/mocks/transformer.go b/pkg/transformers/test_data/mocks/transformer.go new file mode 100644 index 00000000..0b054567 --- /dev/null +++ b/pkg/transformers/test_data/mocks/transformer.go @@ -0,0 +1,44 @@ +package mocks + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type MockTransformer struct { + ExecuteWasCalled bool + ExecuteError error + PassedLogs []types.Log + PassedHeader core.Header + config shared.TransformerConfig +} + +func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { + if mh.ExecuteError != nil { + return mh.ExecuteError + } + mh.ExecuteWasCalled = true + mh.PassedLogs = logs + mh.PassedHeader = header + return nil +} + +func (mh *MockTransformer) GetConfig() shared.TransformerConfig { + return mh.config +} + +func (mh *MockTransformer) SetTransformerConfig(config shared.TransformerConfig) { + mh.config = config +} + +func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared.Transformer { + return mh +} + +var FakeTransformerConfig = shared.TransformerConfig{ + TransformerName: "FakeTransformer", + ContractAddresses: []string{"FakeAddress"}, + Topic: "FakeTopic", +} From a3737c46f63ed92c5cc9f0877035b45a975c5810 Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 13 Dec 2018 13:16:49 +0100 Subject: [PATCH 27/33] Remove duplicate entry in initializer map --- cmd/continuousLogSync.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 9f3e2fec..f6824e2e 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -91,7 +91,6 @@ func getTransformerInitializers(transformerNames []string) []shared2.Transformer func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { initializerMap := make(map[string]shared2.TransformerInitializer) - initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer From 129964f3bcb705e2d95a67efe11d232515216e1b Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 13 Dec 2018 13:58:16 +0100 Subject: [PATCH 28/33] Add tests for now shared MissingHeaders --- .../shared/repository_utility_test.go | 84 ++++++++++++++++++ .../shared_behaviors/repository_behaviors.go | 88 +------------------ 2 files changed, 87 insertions(+), 85 deletions(-) diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go index b24cde03..1561a1ac 100644 --- a/pkg/transformers/shared/repository_utility_test.go +++ b/pkg/transformers/shared/repository_utility_test.go @@ -17,11 +17,95 @@ package shared_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/test_config" + "math/rand" ) var _ = Describe("Repository utilities", func() { + Describe("MissingHeaders", func() { + var ( + db *postgres.DB + repository shared.Repository + repositoryTwo shared.Repository + headerRepository datastore.HeaderRepository + startingBlockNumber int64 + endingBlockNumber int64 + eventSpecificBlockNumber int64 + blockNumbers []int64 + headerIDs []int64 + notCheckedSQL string + err error + ) + + BeforeEach(func() { + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + repository = shared.Repository{} + repositoryTwo = shared.Repository{} + headerRepository = repositories.NewHeaderRepository(db) + + columnNames, err := repository.GetCheckedColumnNames(db) + Expect(err).NotTo(HaveOccurred()) + notCheckedSQL = repository.CreateNotCheckedSQL(columnNames) + + startingBlockNumber = rand.Int63() + eventSpecificBlockNumber = startingBlockNumber + 1 + endingBlockNumber = startingBlockNumber + 2 + outOfRangeBlockNumber := endingBlockNumber + 1 + + blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber} + + headerIDs = []int64{} + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + }) + + It("only treats headers as checked if the event specific logs have been checked", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + for _, n := range blockNumbers { + _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) + Expect(err).NotTo(HaveOccurred()) + } + + Expect(err).NotTo(HaveOccurred()) + nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(3)) + Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) + Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) + Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) + + nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) + Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10))) + Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10))) + }) + }) + Describe("GetCheckedColumnNames", func() { It("gets the column names from checked_headers", func() { db := test_config.NewTestDB(test_config.NewTestNode()) diff --git a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go index e6c24847..745dcd3d 100644 --- a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go +++ b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go @@ -59,7 +59,7 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { var logEventModel = inputs.TestModel BeforeEach(func() { - headerRepository = GetHeaderRepository() + headerRepository = getHeaderRepository() headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) }) @@ -130,95 +130,13 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { }) } -// Saved, will base tests of aggregate fetching on this -/*func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInputs) { - Describe("MissingHeaders", func() { - var ( - repository = inputs.Repository - startingBlockNumber int64 - endingBlockNumber int64 - eventSpecificBlockNumber int64 - blockNumbers []int64 - headerIDs []int64 - ) - - BeforeEach(func() { - headerRepository = GetHeaderRepository() - startingBlockNumber = rand.Int63() - eventSpecificBlockNumber = startingBlockNumber + 1 - endingBlockNumber = startingBlockNumber + 2 - outOfRangeBlockNumber := endingBlockNumber + 1 - - blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber} - - headerIDs = []int64{} - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - }) - - It("returns headers that haven't been checked", func() { - err := repository.MarkHeaderChecked(headerIDs[1]) - Expect(err).NotTo(HaveOccurred()) - - headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) - - Expect(err).NotTo(HaveOccurred()) - Expect(len(headers)).To(Equal(2)) - Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) - Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) - }) - - It("only treats headers as checked if the event specific logs have been checked", func() { - _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) - Expect(err).NotTo(HaveOccurred()) - - headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) - - Expect(err).NotTo(HaveOccurred()) - Expect(len(headers)).To(Equal(3)) - Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) - Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) - Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) - }) - - It("only returns headers associated with the current node", func() { - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - for _, n := range blockNumbers { - _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) - Expect(err).NotTo(HaveOccurred()) - } - repositoryTwo := inputs.RepositoryTwo - repositoryTwo.SetDB(dbTwo) - - err := repository.MarkHeaderChecked(headerIDs[0]) - Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(nodeOneMissingHeaders)).To(Equal(2)) - Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - - nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(startingBlockNumber, endingBlockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) - Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - }) - }) -}*/ - func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehaviorInputs) { var repository = inputs.Repository var checkedHeaderColumn = inputs.CheckedHeaderColumnName Describe("MarkHeaderChecked", func() { BeforeEach(func() { - headerRepository = GetHeaderRepository() + headerRepository = getHeaderRepository() headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) }) @@ -256,7 +174,7 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav }) } -func GetHeaderRepository() repositories.HeaderRepository { +func getHeaderRepository() repositories.HeaderRepository { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) From 35f7f719e640a747ce790ce32edfe841bf58e1ac Mon Sep 17 00:00:00 2001 From: Edvard Date: Thu, 13 Dec 2018 16:08:57 +0100 Subject: [PATCH 29/33] Swap strings.Builder for bytes.Buffer for go 1.9 support --- pkg/transformers/shared/repository.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index 129da6e9..933b999d 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -1,12 +1,12 @@ package shared import ( + "bytes" "database/sql" "database/sql/driver" "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "strings" ) type Repository struct{} @@ -85,18 +85,19 @@ func (_ Repository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { // Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" // [] => "FALSE" func (_ Repository) CreateNotCheckedSQL(boolColumns []string) string { - var result strings.Builder + var result bytes.Buffer if len(boolColumns) == 0 { return "FALSE" } result.WriteString("NOT (") + // Loop excluding last column name for _, column := range boolColumns[:len(boolColumns)-1] { result.WriteString(fmt.Sprintf("%v AND ", column)) } - // No trailing "OR" for last column name + // No trailing "OR" for the last column name result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1])) return result.String() From 7cf694f5e76c0c6eea12e03b4fa26da267c459a0 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 14 Dec 2018 10:30:08 +0100 Subject: [PATCH 30/33] Update transformer documentation --- pkg/transformers/DOCUMENTATION.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index ed3e8a4c..de85a455 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -25,7 +25,7 @@ The transformer process for each of these different log types is the same, excep ## Creating a Transformer 1. Pull an example event (from kovan / ganache etc.) -1. Add event sig to [`constants.go`](./shared/constants.go) +1. Add event & method sig, contract address, `checked_headers` column name, and label to relevant files in [`constants`](./shared/constants) 1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go) 1. Create DB table (using [`create_migration`](../../scripts/create_migration)) 1. Create columns in `checked_headers` in the _same_ migration @@ -36,23 +36,23 @@ The transformer process for each of these different log types is the same, excep 1. Write repository + repository tests 1. Create converter + repository mocks 1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go` -1. Create transformer + transformer tests -1. Wire up transformer in [`transformers.go`](./transformers.go) +1. Wire up transformer in [`transformers.go`](./transformers.go), remembering to add it to `TransformerInitializers()` 1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go) 1. Manually trigger an event and check that it gets persisted to postgres +1. Create an integration test for the shiny new transformer in [`integration_tests`](./integration_tests) **Fetching Logs** 1. Generate an example raw log event, by either: - - Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)). + - Pulling the log directly from the Kovan deployment ([address.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants/address.go)). - Deploying the contract to a local chain and emiting the event manually. 1. Fetch the logs from the chain based on the example event's topic zero: - - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go). - - Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event. - - Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain. + - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants/signature.go`](./shared/constants/signature.go). + - Fetching is done in batch from the [`watcher`](../../libraries/shared/watcher.go). + - The logs are then chunked up by the [`chunker`](./shared/log_chunker.go) before being delegated to each transformer. **Coverting logs** @@ -105,8 +105,7 @@ The transformer process for each of these different log types is the same, excep - The `checked_headers` table allows us to keep track of which headers have been checked for a given log type. - To create a new migration file: `./scripts/create_migration create_flop_kick` - See `db/migrations/1536942529_create_flop_kick.up.sql`. - - The specific log event tables are all created in the `maker` - schema. + - The specific log event tables are all created in the `maker` schema. - There is a one-many association between `headers` and the log event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed. - To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private` @@ -120,7 +119,7 @@ The transformer process for each of these different log types is the same, excep **Wire each component up in the transformer** -- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`. +- We use a [`TransformerInitializer`](./shared/transformer.go) struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. - See any of `pkg/transformers/flop_kick/transformer.go` - All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration. - The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers. From e54699c0396713acc46ee6cc44638e842cebe454 Mon Sep 17 00:00:00 2001 From: Edvard Date: Fri, 14 Dec 2018 15:22:46 +0100 Subject: [PATCH 31/33] Remove todo on chunker non-match (it's basically a filter) --- pkg/transformers/shared/log_chunker.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go index dea03a5b..b7373ff8 100644 --- a/pkg/transformers/shared/log_chunker.go +++ b/pkg/transformers/shared/log_chunker.go @@ -55,7 +55,6 @@ func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { // Topic0 is not unique to each transformer, also need to consider the contract address relevantTransformers := chunker.AddressToNames[log.Address.String()] - // TODO What should happen if log can't be assigned? for _, transformer := range relevantTransformers { if chunker.NameToTopic0[transformer] == log.Topics[0] { chunks[transformer] = append(chunks[transformer], log) From 833dde62cff0e9867e82a1ca545fa3dbd50642bb Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 7 Jan 2019 15:19:31 +0100 Subject: [PATCH 32/33] Remove injection of fetcher and repository to watcher --- cmd/backfillMakerLogs.go | 6 +---- cmd/continuousLogSync.go | 5 +--- environments/staging.toml | 2 +- libraries/shared/watcher.go | 23 +++++++------------ pkg/transformers/bite/repository.go | 4 ++-- .../cat_file/chop_lump/repository.go | 4 ++-- pkg/transformers/cat_file/flip/repository.go | 4 ++-- .../cat_file/pit_vow/repository.go | 4 ++-- pkg/transformers/deal/repository.go | 4 ++-- pkg/transformers/dent/repository.go | 4 ++-- pkg/transformers/drip_drip/repository.go | 4 ++-- pkg/transformers/drip_file/ilk/repository.go | 4 ++-- pkg/transformers/drip_file/repo/repository.go | 4 ++-- pkg/transformers/drip_file/vow/repository.go | 4 ++-- pkg/transformers/flap_kick/repository.go | 4 ++-- pkg/transformers/flip_kick/repository.go | 4 ++-- pkg/transformers/flop_kick/repository.go | 4 ++-- pkg/transformers/frob/repository.go | 4 ++-- .../pit_file/debt_ceiling/repository.go | 4 ++-- pkg/transformers/pit_file/ilk/repository.go | 4 ++-- pkg/transformers/price_feeds/repository.go | 4 ++-- pkg/transformers/shared/repository.go | 12 ++++------ pkg/transformers/tend/repository.go | 4 ++-- pkg/transformers/vat_flux/repository.go | 4 ++-- pkg/transformers/vat_fold/repository.go | 4 ++-- pkg/transformers/vat_grab/repository.go | 4 ++-- pkg/transformers/vat_heal/repository.go | 4 ++-- pkg/transformers/vat_init/repository.go | 4 ++-- pkg/transformers/vat_move/repository.go | 4 ++-- pkg/transformers/vat_slip/repository.go | 4 ++-- pkg/transformers/vat_toll/repository.go | 4 ++-- pkg/transformers/vat_tune/repository.go | 4 ++-- pkg/transformers/vow_flog/repository.go | 4 ++-- 33 files changed, 72 insertions(+), 88 deletions(-) diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 4f19038e..33b9e919 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -17,8 +17,6 @@ package cmd import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers" @@ -47,9 +45,7 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } - repository := &shared2.Repository{} - fetcher := shared2.NewFetcher(blockChain) - watcher := shared.NewWatcher(db, fetcher, repository) + watcher := shared.NewWatcher(db, blockChain) watcher.AddTransformers(transformers.TransformerInitializers()) err = watcher.Execute() diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index f6824e2e..710a6bf4 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -58,12 +58,9 @@ func syncMakerLogs() { log.Fatal("Failed to initialize database.") } - fetcher := shared2.NewFetcher(blockChain) - repository := &shared2.Repository{} - initializers := getTransformerInitializers(transformerNames) - watcher := shared.NewWatcher(db, fetcher, repository) + watcher := shared.NewWatcher(db, blockChain) watcher.AddTransformers(initializers) for range ticker.C { diff --git a/environments/staging.toml b/environments/staging.toml index 6ed252dd..bcdf0079 100644 --- a/environments/staging.toml +++ b/environments/staging.toml @@ -6,7 +6,7 @@ password = "vulcanize" port = 5432 [client] -ipcPath = "http://147.75.199.135:8545" +ipcPath = "http://kovan0.vulcanize.io:8545" [datadog] name = "maker_vdb_staging" diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 7b68752c..8a8ded13 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -8,12 +8,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" ) -type WatcherRepository interface { - GetCheckedColumnNames(db *postgres.DB) ([]string, error) - CreateNotCheckedSQL(boolColumns []string) string - MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) -} - type Watcher struct { Transformers []shared.Transformer DB *postgres.DB @@ -21,16 +15,15 @@ type Watcher struct { Chunker shared.Chunker Addresses []common.Address Topics []common.Hash - Repository WatcherRepository } -func NewWatcher(db *postgres.DB, fetcher shared.LogFetcher, repository WatcherRepository) Watcher { +func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher { chunker := shared.NewLogChunker() + fetcher := shared.NewFetcher(bc) return Watcher{ - DB: db, - Fetcher: fetcher, - Chunker: chunker, - Repository: repository, + DB: db, + Fetcher: fetcher, + Chunker: chunker, } } @@ -58,14 +51,14 @@ func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitial } func (watcher *Watcher) Execute() error { - checkedColumnNames, err := watcher.Repository.GetCheckedColumnNames(watcher.DB) + checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB) if err != nil { return err } - notCheckedSQL := watcher.Repository.CreateNotCheckedSQL(checkedColumnNames) + notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) // TODO Handle start and end numbers in transformers - missingHeaders, err := watcher.Repository.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) + missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) if err != nil { log.Error("Fetching of missing headers failed in watcher!") return err diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 290deaee..1e3d860b 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -52,7 +52,7 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.BiteChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.BiteChecked) if err != nil { tx.Rollback() return err @@ -62,5 +62,5 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er } func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) } diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index c63e83b6..9dff41f7 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -49,7 +49,7 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileChopLumpChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileChopLumpChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte } func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) } func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/cat_file/flip/repository.go b/pkg/transformers/cat_file/flip/repository.go index 624e37d3..a07ca207 100644 --- a/pkg/transformers/cat_file/flip/repository.go +++ b/pkg/transformers/cat_file/flip/repository.go @@ -48,7 +48,7 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileFlipChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFileFlipChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac } func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) } func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/cat_file/pit_vow/repository.go b/pkg/transformers/cat_file/pit_vow/repository.go index 8e2e3a29..861dc2a8 100644 --- a/pkg/transformers/cat_file/pit_vow/repository.go +++ b/pkg/transformers/cat_file/pit_vow/repository.go @@ -48,7 +48,7 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFilePitVowChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.CatFilePitVowChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf } func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) } func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go index 825f00d6..06f23230 100644 --- a/pkg/transformers/deal/repository.go +++ b/pkg/transformers/deal/repository.go @@ -49,7 +49,7 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DealChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DealChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er } func (repository DealRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) } func (repository *DealRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index 80aee3e0..0ff5370b 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -54,7 +54,7 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DentChecked) if err != nil { tx.Rollback() return err @@ -63,7 +63,7 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er } func (repository DentRepository) MarkHeaderChecked(headerId int64) error { - return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) + return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) } func (repository *DentRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index c9fa1623..edb1208a 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -48,7 +48,7 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripDripChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripDripChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{} } func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) } func (repository *DripDripRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 3740c548..117a28fd 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -50,7 +50,7 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileIlkChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileIlkChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac } func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) } func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 23d08af2..dc9589a7 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -50,7 +50,7 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileRepoChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileRepoChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa } func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) } func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 86bdf8f2..aa23bea8 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -49,7 +49,7 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileVowChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.DripFileVowChecked) if err != nil { tx.Rollback() return err @@ -59,7 +59,7 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac } func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) } func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/flap_kick/repository.go b/pkg/transformers/flap_kick/repository.go index 950ebcb0..f9f11c7a 100644 --- a/pkg/transformers/flap_kick/repository.go +++ b/pkg/transformers/flap_kick/repository.go @@ -47,7 +47,7 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{ } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlapKickChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlapKickChecked) if err != nil { tx.Rollback() return err @@ -56,7 +56,7 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{ } func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) } func (repository *FlapKickRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/flip_kick/repository.go b/pkg/transformers/flip_kick/repository.go index 99c376be..db8a8d01 100644 --- a/pkg/transformers/flip_kick/repository.go +++ b/pkg/transformers/flip_kick/repository.go @@ -47,7 +47,7 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{} return err } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlipKickChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlipKickChecked) if err != nil { tx.Rollback() return err @@ -56,7 +56,7 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{} } func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error { - return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) + return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) } func (repository *FlipKickRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/flop_kick/repository.go b/pkg/transformers/flop_kick/repository.go index 1bab62c6..7a302c75 100644 --- a/pkg/transformers/flop_kick/repository.go +++ b/pkg/transformers/flop_kick/repository.go @@ -48,7 +48,7 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{} } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlopKickChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FlopKickChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{} } func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error { - return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) + return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) } func (repository *FlopKickRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index e2c26435..bf6d4d08 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -46,7 +46,7 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er return err } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.FrobChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.FrobChecked) if err != nil { tx.Rollback() return err @@ -55,7 +55,7 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er } func (repository FrobRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) } func (repository *FrobRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 38b648ba..12f69fb2 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -50,7 +50,7 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileDebtCeilingChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileDebtCeilingChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i } func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) } func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index 3dae51c6..9a187de7 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -49,7 +49,7 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileIlkChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.PitFileIlkChecked) if err != nil { tx.Rollback() return err @@ -58,7 +58,7 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface } func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) } func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go index af1caf19..cbb9a012 100644 --- a/pkg/transformers/price_feeds/repository.go +++ b/pkg/transformers/price_feeds/repository.go @@ -45,7 +45,7 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{ } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.PriceFeedsChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.PriceFeedsChecked) if err != nil { tx.Rollback() return err @@ -54,7 +54,7 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{ } func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) } func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index 933b999d..65c01c99 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -9,9 +9,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type Repository struct{} - -func (_ Repository) MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { +func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2) ON CONFLICT (header_id) DO @@ -19,7 +17,7 @@ func (_ Repository) MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHe return err } -func (_ Repository) MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error { +func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2) ON CONFLICT (header_id) DO @@ -28,7 +26,7 @@ func (_ Repository) MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, c } // Treats a header as missing if it's not in the headers table, or not checked for some log type -func (_ Repository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { +func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { var result []core.Header var query string var err error @@ -53,7 +51,7 @@ func (_ Repository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, return result, err } -func (_ Repository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { +func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { // Query returns `[]driver.Value`, nullable polymorphic interface var queryResult []driver.Value columnNamesQuery := @@ -84,7 +82,7 @@ func (_ Repository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { // Defaults to FALSE when no columns are provided. // Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" // [] => "FALSE" -func (_ Repository) CreateNotCheckedSQL(boolColumns []string) string { +func CreateNotCheckedSQL(boolColumns []string) string { var result bytes.Buffer if len(boolColumns) == 0 { diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index bfabd248..17620a00 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -55,7 +55,7 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.TendChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.TendChecked) if err != nil { tx.Rollback() return err @@ -64,7 +64,7 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er } func (repository TendRepository) MarkHeaderChecked(headerId int64) error { - return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) + return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) } func (repository *TendRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index 78089701..2febe527 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -47,7 +47,7 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFluxChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFluxChecked) if err != nil { tx.Rollback() return err @@ -57,7 +57,7 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{}) } func (repository VatFluxRepository) MarkHeaderChecked(headerId int64) error { - return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) + return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) } func (repository *VatFluxRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index 01ba28ba..5d831bca 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -49,7 +49,7 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFoldChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatFoldChecked) if err != nil { tx.Rollback() return err @@ -59,7 +59,7 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{}) } func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) } func (repository *VatFoldRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index 14364b90..a8a274a4 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -34,7 +34,7 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) return err } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatGrabChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatGrabChecked) if err != nil { tx.Rollback() return err @@ -43,7 +43,7 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{}) } func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) } func (repository *VatGrabRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_heal/repository.go b/pkg/transformers/vat_heal/repository.go index 861025a6..aed8fcd0 100644 --- a/pkg/transformers/vat_heal/repository.go +++ b/pkg/transformers/vat_heal/repository.go @@ -52,7 +52,7 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatHealChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatHealChecked) if err != nil { tx.Rollback() return err @@ -61,5 +61,5 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) } func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error { - return shared.Repository{}.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) + return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) } diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index 1bb56d12..14b2d4c8 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -49,7 +49,7 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatInitChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatInitChecked) if err != nil { tx.Rollback() return err @@ -59,7 +59,7 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{}) } func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) } func (repository *VatInitRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index 913b587c..2548c2b4 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -50,7 +50,7 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatMoveChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatMoveChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) } func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) } func (repository *VatMoveRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index b84f248c..15f412ca 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -34,7 +34,7 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatSlipChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatSlipChecked) if err != nil { tx.Rollback() return err @@ -44,7 +44,7 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{}) } func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) } func (repository *VatSlipRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index 6c738e54..2c9be8a4 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -34,7 +34,7 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTollChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTollChecked) if err != nil { tx.Rollback() return err @@ -43,7 +43,7 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{}) } func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) } func (repository *VatTollRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index 79404cb0..c9951ce6 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -34,7 +34,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTuneChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTuneChecked) if err != nil { tx.Rollback() return err @@ -43,7 +43,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{}) } func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) } func (repository *VatTuneRepository) SetDB(db *postgres.DB) { diff --git a/pkg/transformers/vow_flog/repository.go b/pkg/transformers/vow_flog/repository.go index 994de3e2..48076802 100644 --- a/pkg/transformers/vow_flog/repository.go +++ b/pkg/transformers/vow_flog/repository.go @@ -50,7 +50,7 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{}) } } - err = shared.Repository{}.MarkHeaderCheckedInTransaction(headerID, tx, constants.VowFlogChecked) + err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VowFlogChecked) if err != nil { tx.Rollback() return err @@ -60,7 +60,7 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{}) } func (repository VowFlogRepository) MarkHeaderChecked(headerID int64) error { - return shared.Repository{}.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) + return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) } func (repository *VowFlogRepository) SetDB(db *postgres.DB) { From 50cc0c245984a18560d2eae3ce7edb136c648b89 Mon Sep 17 00:00:00 2001 From: Edvard Date: Mon, 7 Jan 2019 15:20:05 +0100 Subject: [PATCH 33/33] Update tests for watcher and repository --- libraries/shared/watcher_test.go | 70 +++++-------------- .../shared/repository_utility_test.go | 22 +++--- 2 files changed, 28 insertions(+), 64 deletions(-) diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index c5ea6836..c93c6336 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -2,6 +2,7 @@ package shared_test import ( "errors" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" @@ -19,19 +20,17 @@ import ( var _ = Describe("Watcher", func() { It("initialises correctly", func() { db := test_config.NewTestDB(core.Node{ID: "testNode"}) - fetcher := &mocks.MockLogFetcher{} - repository := &mocks.MockWatcherRepository{} + bc := fakes.NewMockBlockChain() - watcher := shared.NewWatcher(db, fetcher, repository) + watcher := shared.NewWatcher(db, bc) Expect(watcher.DB).To(Equal(db)) - Expect(watcher.Fetcher).To(Equal(fetcher)) + Expect(watcher.Fetcher).NotTo(BeNil()) Expect(watcher.Chunker).NotTo(BeNil()) - Expect(watcher.Repository).To(Equal(repository)) }) It("adds transformers", func() { - watcher := shared.NewWatcher(nil, nil, nil) + watcher := shared.NewWatcher(nil, nil) fakeTransformer := &mocks.MockTransformer{} fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig) watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) @@ -43,7 +42,7 @@ var _ = Describe("Watcher", func() { }) It("adds transformers from multiple sources", func() { - watcher := shared.NewWatcher(nil, nil, nil) + watcher := shared.NewWatcher(nil, nil) fakeTransformer1 := &mocks.MockTransformer{} fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig) @@ -64,22 +63,22 @@ var _ = Describe("Watcher", func() { var ( db *postgres.DB watcher shared.Watcher + mockBlockChain fakes.MockBlockChain fakeTransformer *mocks.MockTransformer headerRepository repositories.HeaderRepository - mockFetcher mocks.MockLogFetcher repository mocks.MockWatcherRepository ) BeforeEach(func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) - mockFetcher = mocks.MockLogFetcher{} + mockBlockChain = fakes.MockBlockChain{} headerRepository = repositories.NewHeaderRepository(db) _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) repository = mocks.MockWatcherRepository{} - watcher = shared.NewWatcher(db, &mockFetcher, &repository) + watcher = shared.NewWatcher(db, &mockBlockChain) }) It("executes each transformer", func() { @@ -122,10 +121,10 @@ var _ = Describe("Watcher", func() { Topics: []common.Hash{common.HexToHash("0xA")}} logB := types.Log{Address: common.HexToAddress("0xB"), Topics: []common.Hash{common.HexToHash("0xB")}} - mockFetcher.SetFetchedLogs([]types.Log{logA, logB}) + mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB}) repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - watcher = shared.NewWatcher(db, &mockFetcher, &repository) + watcher = shared.NewWatcher(db, &mockBlockChain) watcher.AddTransformers([]shared2.TransformerInitializer{ transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer}) @@ -135,43 +134,6 @@ var _ = Describe("Watcher", func() { Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB})) }) - Describe("uses the repository correctly:", func() { - - It("calls MissingHeaders", func() { - err := watcher.Execute() - Expect(err).To(Not(HaveOccurred())) - Expect(repository.MissingHeadersCalled).To(BeTrue()) - }) - - It("propagates MissingHeaders errors", func() { - missingHeadersError := errors.New("MissingHeadersError") - repository.MissingHeadersError = missingHeadersError - - err := watcher.Execute() - Expect(err).To(MatchError(missingHeadersError)) - }) - - It("calls CreateNotCheckedSQL", func() { - err := watcher.Execute() - Expect(err).NotTo(HaveOccurred()) - Expect(repository.CreateNotCheckedSQLCalled).To(BeTrue()) - }) - - It("calls GetCheckedColumnNames", func() { - err := watcher.Execute() - Expect(err).NotTo(HaveOccurred()) - Expect(repository.GetCheckedColumnNamesCalled).To(BeTrue()) - }) - - It("propagates GetCheckedColumnNames errors", func() { - getCheckedColumnNamesError := errors.New("GetCheckedColumnNamesError") - repository.GetCheckedColumnNamesError = getCheckedColumnNamesError - - err := watcher.Execute() - Expect(err).To(MatchError(getCheckedColumnNamesError)) - }) - }) - Describe("uses the LogFetcher correctly:", func() { BeforeEach(func() { repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) @@ -180,12 +142,18 @@ var _ = Describe("Watcher", func() { It("fetches logs", func() { err := watcher.Execute() Expect(err).NotTo(HaveOccurred()) - Expect(mockFetcher.FetchLogsCalled).To(BeTrue()) + + fakeHash := common.HexToHash(fakes.FakeHeader.Hash) + mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(ethereum.FilterQuery{ + BlockHash: &fakeHash, + Addresses: nil, + Topics: [][]common.Hash{nil}, + }) }) It("propagates log fetcher errors", func() { fetcherError := errors.New("FetcherError") - mockFetcher.SetFetcherError(fetcherError) + mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError) err := watcher.Execute() Expect(err).To(MatchError(fetcherError)) diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go index 1561a1ac..2699108e 100644 --- a/pkg/transformers/shared/repository_utility_test.go +++ b/pkg/transformers/shared/repository_utility_test.go @@ -31,8 +31,6 @@ var _ = Describe("Repository utilities", func() { Describe("MissingHeaders", func() { var ( db *postgres.DB - repository shared.Repository - repositoryTwo shared.Repository headerRepository datastore.HeaderRepository startingBlockNumber int64 endingBlockNumber int64 @@ -46,13 +44,11 @@ var _ = Describe("Repository utilities", func() { BeforeEach(func() { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) - repository = shared.Repository{} - repositoryTwo = shared.Repository{} headerRepository = repositories.NewHeaderRepository(db) - columnNames, err := repository.GetCheckedColumnNames(db) + columnNames, err := shared.GetCheckedColumnNames(db) Expect(err).NotTo(HaveOccurred()) - notCheckedSQL = repository.CreateNotCheckedSQL(columnNames) + notCheckedSQL = shared.CreateNotCheckedSQL(columnNames) startingBlockNumber = rand.Int63() eventSpecificBlockNumber = startingBlockNumber + 1 @@ -73,7 +69,7 @@ var _ = Describe("Repository utilities", func() { _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) Expect(err).NotTo(HaveOccurred()) - headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) + headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) Expect(err).NotTo(HaveOccurred()) Expect(len(headers)).To(Equal(3)) @@ -91,14 +87,14 @@ var _ = Describe("Repository utilities", func() { } Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) + nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeOneMissingHeaders)).To(Equal(3)) Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL) + nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL) Expect(err).NotTo(HaveOccurred()) Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10))) @@ -111,7 +107,7 @@ var _ = Describe("Repository utilities", func() { db := test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) expectedColumnNames := getExpectedColumnNames() - actualColumnNames, err := shared.Repository{}.GetCheckedColumnNames(db) + actualColumnNames, err := shared.GetCheckedColumnNames(db) Expect(err).NotTo(HaveOccurred()) Expect(actualColumnNames).To(Equal(expectedColumnNames)) }) @@ -121,20 +117,20 @@ var _ = Describe("Repository utilities", func() { It("generates a correct SQL string for one column", func() { columns := []string{"columnA"} expected := "NOT (columnA)" - actual := shared.Repository{}.CreateNotCheckedSQL(columns) + actual := shared.CreateNotCheckedSQL(columns) Expect(actual).To(Equal(expected)) }) It("generates a correct SQL string for several columns", func() { columns := []string{"columnA", "columnB"} expected := "NOT (columnA AND columnB)" - actual := shared.Repository{}.CreateNotCheckedSQL(columns) + actual := shared.CreateNotCheckedSQL(columns) Expect(actual).To(Equal(expected)) }) It("defaults to FALSE when there are no columns", func() { expected := "FALSE" - actual := shared.Repository{}.CreateNotCheckedSQL([]string{}) + actual := shared.CreateNotCheckedSQL([]string{}) Expect(actual).To(Equal(expected)) }) })