diff --git a/cmd/backfillMakerLogs.go b/cmd/backfillMakerLogs.go index 07a3d003..33b9e919 100644 --- a/cmd/backfillMakerLogs.go +++ b/cmd/backfillMakerLogs.go @@ -17,7 +17,6 @@ package cmd import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers" @@ -46,13 +45,13 @@ func backfillMakerLogs() { log.Fatal("Failed to initialize database.") } - watcher := shared.Watcher{ - DB: *db, - Blockchain: blockChain, - } + watcher := shared.NewWatcher(db, blockChain) watcher.AddTransformers(transformers.TransformerInitializers()) - watcher.Execute() + err = watcher.Execute() + if err != nil { + // TODO Handle watcher error in backfillMakerLogs + } } func init() { diff --git a/cmd/continuousLogSync.go b/cmd/continuousLogSync.go index 9c23af1b..710a6bf4 100644 --- a/cmd/continuousLogSync.go +++ b/cmd/continuousLogSync.go @@ -58,70 +58,66 @@ func syncMakerLogs() { log.Fatal("Failed to initialize database.") } - watcher := shared.Watcher{ - DB: *db, - Blockchain: blockChain, - } + initializers := getTransformerInitializers(transformerNames) - transformerInititalizers := getTransformerInititalizers(transformerNames) - watcher.AddTransformers(transformerInititalizers) + watcher := shared.NewWatcher(db, blockChain) + watcher.AddTransformers(initializers) for range ticker.C { - watcher.Execute() + err = watcher.Execute() + if err != nil { + // TODO Handle watcher errors in ContinuousLogSync + } } } -func getTransformerInititalizers(transformerNames []string) []shared2.TransformerInitializer { - transformerInitializerMap := buildTransformerInitializerMap() - var transformerInitializers []shared2.TransformerInitializer +func getTransformerInitializers(transformerNames []string) []shared2.TransformerInitializer { + var initializers []shared2.TransformerInitializer if transformerNames[0] == "all" { - for _, v := range transformerInitializerMap { - transformerInitializers = append(transformerInitializers, v) - } + initializers = transformers.TransformerInitializers() } else { + initializerMap := buildTransformerInitializerMap() for _, transformerName := range transformerNames { - initializer := transformerInitializerMap[transformerName] - transformerInitializers = append(transformerInitializers, initializer) + initializers = append(initializers, initializerMap[transformerName]) } } - - return transformerInitializers + return initializers } func buildTransformerInitializerMap() map[string]shared2.TransformerInitializer { - transformerInitializerMap := make(map[string]shared2.TransformerInitializer) + initializerMap := make(map[string]shared2.TransformerInitializer) - transformerInitializerMap[constants.BiteLabel] = transformers.BiteTransformerInitializer - transformerInitializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformerInitializer - transformerInitializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformerInitializer - transformerInitializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformerInitializer - transformerInitializerMap[constants.DealLabel] = transformers.DealTransformerInitializer - transformerInitializerMap[constants.DentLabel] = transformers.DentTransformerInitializer - transformerInitializerMap[constants.DripDripLabel] = transformers.DripDripTransformerInitializer - transformerInitializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformerInitializer - transformerInitializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformerInitializer - transformerInitializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromerInitializer - transformerInitializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformerInitializer - transformerInitializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformerInitializer - transformerInitializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformerInitializer - transformerInitializerMap[constants.FrobLabel] = transformers.FrobTransformerInitializer - transformerInitializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformerInitializer - transformerInitializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformerInitializer - transformerInitializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformerInitializer - transformerInitializerMap[constants.TendLabel] = transformers.TendTransformerInitializer - transformerInitializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformerInitializer - transformerInitializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformerInitializer - transformerInitializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformerInitializer - transformerInitializerMap[constants.VatHealLabel] = transformers.VatHealTransformerInitializer - transformerInitializerMap[constants.VatInitLabel] = transformers.VatInitTransformerInitializer - transformerInitializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformerInitializer - transformerInitializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformerInitializer - transformerInitializerMap[constants.VatTollLabel] = transformers.VatTollTransformerInitializer - transformerInitializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformerInitializer - transformerInitializerMap[constants.VowFlogLabel] = transformers.FlogTransformerInitializer + initializerMap[constants.BiteLabel] = transformers.BiteTransformer.NewTransformer + initializerMap[constants.CatFileChopLumpLabel] = transformers.CatFileChopLumpTransformer.NewLogNoteTransformer + initializerMap[constants.CatFileFlipLabel] = transformers.CatFileFlipTransformer.NewLogNoteTransformer + initializerMap[constants.CatFilePitVowLabel] = transformers.CatFilePitVowTransformer.NewLogNoteTransformer + initializerMap[constants.DealLabel] = transformers.DealTransformer.NewLogNoteTransformer + initializerMap[constants.DentLabel] = transformers.DentTransformer.NewLogNoteTransformer + initializerMap[constants.DripDripLabel] = transformers.DripDripTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileIlkLabel] = transformers.DripFileIlkTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileRepoLabel] = transformers.DripFileRepoTransformer.NewLogNoteTransformer + initializerMap[constants.DripFileVowLabel] = transformers.DripFileVowTransfromer.NewLogNoteTransformer + initializerMap[constants.FlapKickLabel] = transformers.FlapKickTransformer.NewTransformer + initializerMap[constants.FlipKickLabel] = transformers.FlipKickTransformer.NewTransformer + initializerMap[constants.FlopKickLabel] = transformers.FlopKickTransformer.NewTransformer + initializerMap[constants.FrobLabel] = transformers.FrobTransformer.NewTransformer + initializerMap[constants.PitFileDebtCeilingLabel] = transformers.PitFileDebtCeilingTransformer.NewLogNoteTransformer + initializerMap[constants.PitFileIlkLabel] = transformers.PitFileIlkTransformer.NewLogNoteTransformer + initializerMap[constants.PriceFeedLabel] = transformers.PriceFeedTransformer.NewLogNoteTransformer + initializerMap[constants.TendLabel] = transformers.TendTransformer.NewLogNoteTransformer + initializerMap[constants.VatFluxLabel] = transformers.VatFluxTransformer.NewLogNoteTransformer + initializerMap[constants.VatFoldLabel] = transformers.VatFoldTransformer.NewLogNoteTransformer + initializerMap[constants.VatGrabLabel] = transformers.VatGrabTransformer.NewLogNoteTransformer + initializerMap[constants.VatHealLabel] = transformers.VatHealTransformer.NewLogNoteTransformer + initializerMap[constants.VatInitLabel] = transformers.VatInitTransformer.NewLogNoteTransformer + initializerMap[constants.VatMoveLabel] = transformers.VatMoveTransformer.NewLogNoteTransformer + initializerMap[constants.VatSlipLabel] = transformers.VatSlipTransformer.NewLogNoteTransformer + initializerMap[constants.VatTollLabel] = transformers.VatTollTransformer.NewLogNoteTransformer + initializerMap[constants.VatTuneLabel] = transformers.VatTuneTransformer.NewLogNoteTransformer + initializerMap[constants.VowFlogLabel] = transformers.FlogTransformer.NewLogNoteTransformer - return transformerInitializerMap + return initializerMap } func init() { diff --git a/cmd/erc20.go b/cmd/erc20.go deleted file mode 100644 index d0d8f112..00000000 --- a/cmd/erc20.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "time" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/libraries/shared" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" -) - -// erc20Cmd represents the erc20 command -var erc20Cmd = &cobra.Command{ - Use: "erc20", - Short: "Fetches and persists token supply", - Long: `Fetches the totalSupply for the configured token from each block and persists it in Vulcanize DB. -vulcanizedb erc20 --config environments/public - -Expects an ethereum node to be running and requires a .toml config file: - - [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - - [client] - ipcPath = "/Users/user/Library/Ethereum/geth.ipc" -`, - Run: func(cmd *cobra.Command, args []string) { - watchERC20s() - }, -} - -func watchERC20s() { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - blockChain := getBlockChain() - db, err := postgres.NewDB(databaseConfig, blockChain.Node()) - if err != nil { - log.Fatal("Failed to initialize database.") - } - watcher := shared.Watcher{ - DB: *db, - Blockchain: blockChain, - } - - watcher.AddTransformers(every_block.TransformerInitializers()) - for range ticker.C { - watcher.Execute() - } -} - -func init() { - rootCmd.AddCommand(erc20Cmd) -} diff --git a/examples/constants/constants.go b/examples/constants/constants.go deleted file mode 100644 index abc84f94..00000000 --- a/examples/constants/constants.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package constants - -//Contract Addresses -var DaiContractAddress = "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" - -//Abis -var DaiAbiString = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"stop","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"owner_","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"name_","type":"bytes32"}],"name":"setName","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"src","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"stopped","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"authority_","type":"address"}],"name":"setAuthority","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"bytes32"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"},{"name":"wad","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"wad","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"transfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"push","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"dst","type":"address"},{"name":"wad","type":"uint256"}],"name":"move","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"start","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"authority","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"guy","type":"address"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"src","type":"address"},{"name":"guy","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"src","type":"address"},{"name":"wad","type":"uint256"}],"name":"pull","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"symbol_","type":"bytes32"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"authority","type":"address"}],"name":"LogSetAuthority","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"}],"name":"LogSetOwner","type":"event"},{"anonymous":true,"inputs":[{"indexed":true,"name":"sig","type":"bytes4"},{"indexed":true,"name":"guy","type":"address"},{"indexed":true,"name":"foo","type":"bytes32"},{"indexed":true,"name":"bar","type":"bytes32"},{"indexed":false,"name":"wad","type":"uint256"},{"indexed":false,"name":"fax","type":"bytes"}],"name":"LogNote","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"guy","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"src","type":"address"},{"indexed":true,"name":"dst","type":"address"},{"indexed":false,"name":"wad","type":"uint256"}],"name":"Transfer","type":"event"}]` diff --git a/examples/erc20_test_helpers/database.go b/examples/erc20_test_helpers/database.go deleted file mode 100644 index 15946db1..00000000 --- a/examples/erc20_test_helpers/database.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package erc20_test_helpers - -type TokenSupplyDBRow struct { - ID int64 - Supply int64 - BlockID int64 `db:"block_id"` - TokenAddress string `db:"token_address"` -} - -type TransferDBRow struct { - ID int64 `db:"id"` - VulcanizeLogID int64 `db:"vulcanize_log_id"` -} diff --git a/examples/erc20_test_helpers/mocks/mocks.go b/examples/erc20_test_helpers/mocks/mocks.go deleted file mode 100644 index b2944170..00000000 --- a/examples/erc20_test_helpers/mocks/mocks.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mocks - -import ( - "math/big" - - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/fakes" -) - -type Fetcher struct { - ContractAddress string - Abi string - FetchedBlocks []int64 - BlockChain core.BlockChain - supply big.Int -} - -func (f *Fetcher) SetSupply(supply string) { - f.supply.SetString(supply, 10) -} - -func (f Fetcher) GetBlockChain() core.BlockChain { - return f.BlockChain -} - -func (f *Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) { - f.Abi = contractAbi - f.ContractAddress = contractAddress - f.FetchedBlocks = append(f.FetchedBlocks, blockNumber) - - accumulator := big.NewInt(1) - f.supply.Add(&f.supply, accumulator) - - return f.supply, nil -} - -type TotalSupplyRepository struct { - TotalSuppliesCreated []every_block.TokenSupply - MissingBlockNumbers []int64 - StartingBlock int64 - EndingBlock int64 -} - -func (fr *TotalSupplyRepository) Create(supply every_block.TokenSupply) error { - fr.TotalSuppliesCreated = append(fr.TotalSuppliesCreated, supply) - return nil -} - -func (fr *TotalSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { - fr.StartingBlock = startingBlock - fr.EndingBlock = highestBlock - return fr.MissingBlockNumbers, nil -} - -func (fr *TotalSupplyRepository) SetMissingBlocks(missingBlocks []int64) { - fr.MissingBlockNumbers = missingBlocks -} - -type FailureRepository struct { - createFail bool - missingBlocksFail bool - missingBlocksNumbers []int64 -} - -func (fr *FailureRepository) Create(supply every_block.TokenSupply) error { - if fr.createFail { - return fakes.FakeError - } else { - return nil - } -} - -func (fr *FailureRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { - if fr.missingBlocksFail { - return []int64{}, fakes.FakeError - } else { - return fr.missingBlocksNumbers, nil - } -} - -func (fr *FailureRepository) SetCreateFail(fail bool) { - fr.createFail = fail -} - -func (fr *FailureRepository) SetMissingBlocksFail(fail bool) { - fr.missingBlocksFail = fail -} - -func (fr *FailureRepository) SetMissingBlocks(missingBlocks []int64) { - fr.missingBlocksNumbers = missingBlocks -} diff --git a/examples/erc20_watcher/ERC20WatcherREADME.md b/examples/erc20_watcher/ERC20WatcherREADME.md deleted file mode 100644 index 97ecdfe4..00000000 --- a/examples/erc20_watcher/ERC20WatcherREADME.md +++ /dev/null @@ -1,19 +0,0 @@ -# ERC20 Transformers - -## Description -The Transformers in this directory are associated with contract functions and events that conform to the [ERC20 Token interface](https://theethereum.wiki/w/index.php/ERC20_Token_Standard#The_ERC20_Token_Standard_Interface). - -See `libraries/shared/TransformerREADME.md` for further information regarding the Transformer interface. - -## Configuration -In addition to environment configuration mentioned in the main VulcanizeDB README, the ERC20 transformers also need to be configured with contract information for the desired token(s) to be watched. This configuration file is located at `./vulcanizedb/examples/erc20_watcher/config.go`. - -## ERC20 Functions -The `everyblock` directory contains transformers that fetch data from the contract itself, via one of the standard functions. - -Currently, the `totalSupply` function transformer has been implemented. This transformer will fetch the total supply for the given contract address and persist `total_supply` records in the database. - -## Running the transformers -1. If running against a local node, make sure that the node has been started. -1. In a separate terminal run the following command: -`./vulcanizedb erc20 --config ` diff --git a/examples/erc20_watcher/config.go b/examples/erc20_watcher/config.go deleted file mode 100644 index 880d2143..00000000 --- a/examples/erc20_watcher/config.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package erc20_watcher - -import "github.com/vulcanize/vulcanizedb/examples/constants" - -type ContractConfig struct { - Address string - Abi string - FirstBlock int64 - LastBlock int64 - Name string -} - -var DaiConfig = ContractConfig{ - Address: constants.DaiContractAddress, - Abi: constants.DaiAbiString, - FirstBlock: int64(4752008), - LastBlock: -1, - Name: "Dai", -} diff --git a/examples/erc20_watcher/every_block/every_block_suite_test.go b/examples/erc20_watcher/every_block/every_block_suite_test.go deleted file mode 100644 index 69d12677..00000000 --- a/examples/erc20_watcher/every_block/every_block_suite_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - log "github.com/sirupsen/logrus" - "io/ioutil" -) - -func TestEveryBlock(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "EveryBlock Suite") -} - -var _ = BeforeSuite(func() { - log.SetOutput(ioutil.Discard) -}) diff --git a/examples/erc20_watcher/every_block/fetcher.go b/examples/erc20_watcher/every_block/fetcher.go deleted file mode 100644 index 07681fe4..00000000 --- a/examples/erc20_watcher/every_block/fetcher.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "fmt" - log "github.com/sirupsen/logrus" - "math/big" - - "github.com/vulcanize/vulcanizedb/pkg/core" -) - -type ERC20FetcherInterface interface { - FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) - GetBlockChain() core.BlockChain -} - -func NewFetcher(blockchain core.BlockChain) Fetcher { - return Fetcher{ - Blockchain: blockchain, - } -} - -type Fetcher struct { - Blockchain core.BlockChain - ContractAbi string - ContractAddress string -} - -type fetcherError struct { - err string - fetchMethod string -} - -func (fe *fetcherError) Error() string { - return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err) -} - -func newFetcherError(err error, fetchMethod string) *fetcherError { - e := fetcherError{err.Error(), fetchMethod} - log.Info(e.Error()) - return &e -} - -func (f Fetcher) FetchSupplyOf(contractAbi string, contractAddress string, blockNumber int64) (big.Int, error) { - method := "totalSupply" - var result = new(big.Int) - err := f.Blockchain.FetchContractData(contractAbi, contractAddress, method, nil, &result, blockNumber) - - if err != nil { - return *result, newFetcherError(err, method) - } - - return *result, nil -} - -func (f Fetcher) GetBlockChain() core.BlockChain { - return f.Blockchain -} diff --git a/examples/erc20_watcher/every_block/fetcher_test.go b/examples/erc20_watcher/every_block/fetcher_test.go deleted file mode 100644 index 9a51fcc1..00000000 --- a/examples/erc20_watcher/every_block/fetcher_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/big" - - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/geth/client" - rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" - "github.com/vulcanize/vulcanizedb/pkg/geth/node" -) - -var _ = Describe("ERC20 Fetcher", func() { - blockNumber := int64(5502914) - - Describe("FetchSupplyOf", func() { - It("fetches data from the blockchain with the correct arguments", func() { - fakeBlockchain := fakes.NewMockBlockChain() - testFetcher := every_block.NewFetcher(fakeBlockchain) - testAbi := "testAbi" - testContractAddress := "testContractAddress" - _, err := testFetcher.FetchSupplyOf(testAbi, testContractAddress, blockNumber) - - Expect(err).NotTo(HaveOccurred()) - expectedResult := big.Int{} - expected := &expectedResult - fakeBlockchain.AssertFetchContractDataCalledWith(testAbi, testContractAddress, "totalSupply", nil, &expected, blockNumber) - }) - - It("fetches a token's total supply at the given block height", func() { - infuraIPC := "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" - rawRpcClient, err := rpc.Dial(infuraIPC) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) - ethClient := ethclient.NewClient(rawRpcClient) - blockChainClient := client.NewEthClient(ethClient) - node := node.MakeNode(rpcClient) - transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) - realFetcher := every_block.NewFetcher(blockChain) - result, err := realFetcher.FetchSupplyOf(constants.DaiAbiString, constants.DaiContractAddress, blockNumber) - - Expect(err).NotTo(HaveOccurred()) - expectedResult := big.Int{} - expectedResult.SetString("27647235749155415536952630", 10) - Expect(result).To(Equal(expectedResult)) - }) - - It("returns an error if the call to the blockchain fails", func() { - blockChain := fakes.NewMockBlockChain() - blockChain.SetFetchContractDataErr(fakes.FakeError) - errorFetcher := every_block.NewFetcher(blockChain) - result, err := errorFetcher.FetchSupplyOf("", "", 0) - - Expect(result.String()).To(Equal("0")) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("totalSupply")) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - }) - }) -}) diff --git a/examples/erc20_watcher/every_block/integration_test.go b/examples/erc20_watcher/every_block/integration_test.go deleted file mode 100644 index ed13725a..00000000 --- a/examples/erc20_watcher/every_block/integration_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/big" - "strconv" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/test_config" -) - -func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) { - blockNumberString := strconv.FormatInt(blockNumber, 10) - lastBlockOnChain := big.Int{} - lastBlockOnChain.SetString(blockNumberString, 10) - blockChain.SetLastBlock(&lastBlockOnChain) -} - -var _ = Describe("Everyblock transformers", func() { - var db *postgres.DB - var blockChain *fakes.MockBlockChain - var blockNumber int64 - var blockId int64 - var err error - - BeforeEach(func() { - blockChain = fakes.NewMockBlockChain() - blockNumber = erc20_watcher.DaiConfig.FirstBlock - lastBlockNumber := blockNumber + 1 - node := test_config.NewTestNode() - db = test_config.NewTestDB(node) - test_config.CleanTestDB(db) - setLastBlockOnChain(blockChain, lastBlockNumber) - - blockRepository := repositories.NewBlockRepository(db) - - blockId, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber}) - Expect(err).NotTo(HaveOccurred()) - _, err = blockRepository.CreateOrUpdateBlock(core.Block{Number: lastBlockNumber}) - Expect(err).NotTo(HaveOccurred()) - }) - - It("creates a token_supply record for each block in the given range", func() { - initializer := every_block.TokenSupplyTransformerInitializer{Config: erc20_watcher.DaiConfig} - transformer := initializer.NewTokenSupplyTransformer(db, blockChain) - transformer.Execute() - - var tokenSupplyCount int - err := db.QueryRow(`SELECT COUNT(*) FROM token_supply where block_id = $1`, blockId).Scan(&tokenSupplyCount) - Expect(err).ToNot(HaveOccurred()) - Expect(tokenSupplyCount).To(Equal(1)) - - var tokenSupply erc20_test_helpers.TokenSupplyDBRow - err = db.Get(&tokenSupply, `SELECT * from token_supply where block_id = $1`, blockId) - Expect(err).ToNot(HaveOccurred()) - Expect(tokenSupply.BlockID).To(Equal(blockId)) - Expect(tokenSupply.TokenAddress).To(Equal(constants.DaiContractAddress)) - Expect(tokenSupply.Supply).To(Equal(int64(0))) - }) -}) diff --git a/examples/erc20_watcher/every_block/model.go b/examples/erc20_watcher/every_block/model.go deleted file mode 100644 index fc973b9b..00000000 --- a/examples/erc20_watcher/every_block/model.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -type TokenSupply struct { - Value string - TokenAddress string - BlockNumber int64 -} diff --git a/examples/erc20_watcher/every_block/repository.go b/examples/erc20_watcher/every_block/repository.go deleted file mode 100644 index c641665b..00000000 --- a/examples/erc20_watcher/every_block/repository.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "fmt" - log "github.com/sirupsen/logrus" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" -) - -type ERC20RepositoryInterface interface { - Create(supply TokenSupply) error - MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) -} - -type TokenSupplyRepository struct { - *postgres.DB -} - -type repositoryError struct { - err string - msg string - blockNumber int64 -} - -func (re *repositoryError) Error() string { - return fmt.Sprintf(re.msg, re.blockNumber, re.err) -} - -func newRepositoryError(err error, msg string, blockNumber int64) error { - e := repositoryError{err.Error(), msg, blockNumber} - log.Println(e.Error()) - return &e -} - -const ( - GetBlockError = "Error fetching block number %d: %s" - InsertTokenSupplyError = "Error inserting token_supply for block number %d: %s" - MissingBlockError = "Error finding missing token_supply records starting at block %d: %s" -) - -func (tsp *TokenSupplyRepository) Create(supply TokenSupply) error { - var blockId int - err := tsp.DB.Get(&blockId, `SELECT id FROM blocks WHERE number = $1 AND eth_node_id = $2`, supply.BlockNumber, tsp.NodeID) - if err != nil { - return newRepositoryError(err, GetBlockError, supply.BlockNumber) - } - - _, err = tsp.DB.Exec( - `INSERT INTO token_supply (supply, token_address, block_id) - VALUES($1, $2, $3)`, - supply.Value, supply.TokenAddress, blockId) - if err != nil { - return newRepositoryError(err, InsertTokenSupplyError, supply.BlockNumber) - } - return nil -} - -func (tsp *TokenSupplyRepository) MissingBlocks(startingBlock int64, highestBlock int64) ([]int64, error) { - blockNumbers := make([]int64, 0) - - err := tsp.DB.Select( - &blockNumbers, - `SELECT number FROM BLOCKS - LEFT JOIN token_supply ON blocks.id = block_id - WHERE block_id ISNULL - AND eth_node_id = $1 - AND number >= $2 - AND number <= $3 - LIMIT 20`, - tsp.NodeID, - startingBlock, - highestBlock, - ) - if err != nil { - return []int64{}, newRepositoryError(err, MissingBlockError, startingBlock) - } - return blockNumbers, err -} diff --git a/examples/erc20_watcher/every_block/repository_test.go b/examples/erc20_watcher/every_block/repository_test.go deleted file mode 100644 index a508acd1..00000000 --- a/examples/erc20_watcher/every_block/repository_test.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/rand" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/test_config" -) - -var _ = Describe("ERC20 Token Repository", func() { - var db *postgres.DB - var blockId int64 - var blockNumber int64 - var repository every_block.TokenSupplyRepository - var blockRepository repositories.BlockRepository - testAddress := "abc" - - BeforeEach(func() { - node := test_config.NewTestNode() - db = test_config.NewTestDB(node) - test_config.CleanTestDB(db) - repository = every_block.TokenSupplyRepository{DB: db} - blockRepository = *repositories.NewBlockRepository(db) - blockNumber = rand.Int63() - blockId = test_config.NewTestBlock(blockNumber, blockRepository) - }) - - Describe("Create", func() { - It("creates a token supply record", func() { - supply := supplyModel(blockNumber, testAddress, "100") - err := repository.Create(supply) - Expect(err).NotTo(HaveOccurred()) - - dbResult := erc20_test_helpers.TokenSupplyDBRow{} - expectedTokenSupply := erc20_test_helpers.TokenSupplyDBRow{ - Supply: int64(100), - BlockID: blockId, - TokenAddress: testAddress, - } - - var count int - err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(1)) - - err = repository.DB.QueryRowx(`SELECT * FROM token_supply`).StructScan(&dbResult) - Expect(err).NotTo(HaveOccurred()) - Expect(dbResult.Supply).To(Equal(expectedTokenSupply.Supply)) - Expect(dbResult.BlockID).To(Equal(expectedTokenSupply.BlockID)) - Expect(dbResult.TokenAddress).To(Equal(expectedTokenSupply.TokenAddress)) - }) - - It("returns an error if fetching the block's id from the database fails", func() { - errorSupply := supplyModel(-1, "", "") - err := repository.Create(errorSupply) - - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("sql")) - Expect(err.Error()).To(ContainSubstring("block number -1")) - }) - - It("returns an error if inserting the token_supply fails", func() { - errorSupply := supplyModel(blockNumber, "", "") - err := repository.Create(errorSupply) - - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("pq")) - Expect(err.Error()).To(ContainSubstring("token_supply for block number")) - }) - }) - - Describe("When there are multiple nodes", func() { - var node2DB *postgres.DB - var node2BlockRepo *repositories.BlockRepository - var node2BlockId int64 - var node2TokenSupplyRepo every_block.TokenSupplyRepository - var tokenSupply every_block.TokenSupply - - BeforeEach(func() { - node2DB = createDbForAnotherNode() - - //create another block with the same number on node2 - node2BlockRepo = repositories.NewBlockRepository(node2DB) - node2BlockId = test_config.NewTestBlock(blockNumber, *node2BlockRepo) - - tokenSupply = supplyModel(blockNumber, "abc", "100") - node2TokenSupplyRepo = every_block.TokenSupplyRepository{DB: node2DB} - }) - - It("only creates token_supply records for the current node (node2)", func() { - err := node2TokenSupplyRepo.Create(tokenSupply) - Expect(err).NotTo(HaveOccurred()) - - var tokenSupplies []erc20_test_helpers.TokenSupplyDBRow - err = node2TokenSupplyRepo.DB.Select(&tokenSupplies, `SELECT * FROM token_supply`) - Expect(err).NotTo(HaveOccurred()) - Expect(len(tokenSupplies)).To(Equal(1)) - Expect(tokenSupplies[0].BlockID).To(Equal(node2BlockId)) - }) - - It("only includes missing block numbers for the current node", func() { - //create token_supply on original node - err := repository.Create(tokenSupply) - Expect(err).NotTo(HaveOccurred()) - - originalNodeMissingBlocks, err := repository.MissingBlocks(blockNumber, blockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(originalNodeMissingBlocks)).To(Equal(0)) - - node2MissingBlocks, err := node2TokenSupplyRepo.MissingBlocks(blockNumber, blockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(node2MissingBlocks)).To(Equal(1)) - }) - }) - - Describe("MissingBlocks", func() { - It("returns the block numbers for which an associated TokenSupply record hasn't been created", func() { - createTokenSupplyFor(repository, blockNumber) - - newBlockNumber := blockNumber + 1 - test_config.NewTestBlock(newBlockNumber, blockRepository) - blocks, err := repository.MissingBlocks(blockNumber, newBlockNumber) - - Expect(blocks).To(ConsistOf(newBlockNumber)) - Expect(err).NotTo(HaveOccurred()) - }) - - It("only returns blocks within the given range", func() { - newBlockNumber := blockNumber + 1 - test_config.NewTestBlock(newBlockNumber, blockRepository) - blocks, err := repository.MissingBlocks(blockNumber, blockNumber) - - Expect(blocks).NotTo(ConsistOf(newBlockNumber)) - Expect(err).NotTo(HaveOccurred()) - }) - - It("does not return numbers that already have an associated TokenSupply record", func() { - createTokenSupplyFor(repository, blockNumber) - blocks, err := repository.MissingBlocks(blockNumber, blockNumber) - - Expect(blocks).To(BeEmpty()) - Expect(err).NotTo(HaveOccurred()) - }) - }) - - It("deletes the token supply record when the associated block is deleted", func() { - err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"}) - Expect(err).NotTo(HaveOccurred()) - - var count int - err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(1)) - - _, err = db.Query(`DELETE FROM blocks`) - Expect(err).NotTo(HaveOccurred()) - - err = repository.DB.QueryRowx(`SELECT count(*) FROM token_supply`).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(0)) - }) -}) - -func supplyModel(blockNumber int64, tokenAddress string, supplyValue string) every_block.TokenSupply { - return every_block.TokenSupply{ - Value: supplyValue, - TokenAddress: tokenAddress, - BlockNumber: int64(blockNumber), - } -} - -func createTokenSupplyFor(repository every_block.TokenSupplyRepository, blockNumber int64) { - err := repository.Create(every_block.TokenSupply{BlockNumber: blockNumber, Value: "0"}) - Expect(err).NotTo(HaveOccurred()) -} - -func createDbForAnotherNode() *postgres.DB { - anotherNode := core.Node{ - GenesisBlock: "GENESIS", - NetworkID: 1, - ID: "testNodeId", - ClientName: "Geth/v1.7.2-stable-1db4ecdc/darwin-amd64/go1.9", - } - - return test_config.NewTestDB(anotherNode) -} diff --git a/examples/erc20_watcher/every_block/transformer.go b/examples/erc20_watcher/every_block/transformer.go deleted file mode 100644 index 81ff4370..00000000 --- a/examples/erc20_watcher/every_block/transformer.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "fmt" - "math/big" - - log "github.com/sirupsen/logrus" - - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -type Transformer struct { - Fetcher ERC20FetcherInterface - Repository ERC20RepositoryInterface - Config erc20_watcher.ContractConfig -} - -func (t *Transformer) SetConfiguration(config erc20_watcher.ContractConfig) { - t.Config = config -} - -type TokenSupplyTransformerInitializer struct { - Config erc20_watcher.ContractConfig -} - -func (i TokenSupplyTransformerInitializer) NewTokenSupplyTransformer(db *postgres.DB, blockchain core.BlockChain) shared.Transformer { - fetcher := NewFetcher(blockchain) - repository := TokenSupplyRepository{DB: db} - transformer := Transformer{ - Fetcher: &fetcher, - Repository: &repository, - Config: i.Config, - } - - return transformer -} - -const ( - FetchingBlocksError = "Error fetching missing blocks starting at block number %d: %s" - FetchingSupplyError = "Error fetching supply for block %d: %s" - CreateSupplyError = "Error inserting token_supply for block %d: %s" -) - -type transformerError struct { - err string - blockNumber int64 - msg string -} - -func (te *transformerError) Error() string { - return fmt.Sprintf(te.msg, te.blockNumber, te.err) -} - -func newTransformerError(err error, blockNumber int64, msg string) error { - e := transformerError{err.Error(), blockNumber, msg} - log.Println(e.Error()) - return &e -} - -func (t Transformer) Execute() error { - var upperBoundBlock int64 - blockchain := t.Fetcher.GetBlockChain() - lastBlock := blockchain.LastBlock().Int64() - - if t.Config.LastBlock == -1 { - upperBoundBlock = lastBlock - } else { - upperBoundBlock = t.Config.LastBlock - } - - blocks, err := t.Repository.MissingBlocks(t.Config.FirstBlock, upperBoundBlock) - - if err != nil { - return newTransformerError(err, t.Config.FirstBlock, FetchingBlocksError) - } - - log.Printf("Fetching totalSupply for %d blocks", len(blocks)) - for _, blockNumber := range blocks { - totalSupply, err := t.Fetcher.FetchSupplyOf(t.Config.Abi, t.Config.Address, blockNumber) - - if err != nil { - return newTransformerError(err, blockNumber, FetchingSupplyError) - } - model := createTokenSupplyModel(totalSupply, t.Config.Address, blockNumber) - err = t.Repository.Create(model) - - if err != nil { - return newTransformerError(err, blockNumber, CreateSupplyError) - } - } - - return nil -} - -func createTokenSupplyModel(totalSupply big.Int, address string, blockNumber int64) TokenSupply { - return TokenSupply{ - Value: totalSupply.String(), - TokenAddress: address, - BlockNumber: blockNumber, - } -} diff --git a/examples/erc20_watcher/every_block/transformer_test.go b/examples/erc20_watcher/every_block/transformer_test.go deleted file mode 100644 index 17439478..00000000 --- a/examples/erc20_watcher/every_block/transformer_test.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block_test - -import ( - "math/big" - "math/rand" - "strconv" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/examples/constants" - "github.com/vulcanize/vulcanizedb/examples/erc20_test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" - "github.com/vulcanize/vulcanizedb/pkg/fakes" -) - -var testContractConfig = erc20_watcher.ContractConfig{ - Address: constants.DaiContractAddress, - Abi: constants.DaiAbiString, - FirstBlock: int64(4752008), - LastBlock: int64(5750050), - Name: "Dai", -} - -var config = testContractConfig - -var _ = Describe("Everyblock transformer", func() { - var fetcher mocks.Fetcher - var repository mocks.TotalSupplyRepository - var transformer every_block.Transformer - var blockChain *fakes.MockBlockChain - var initialSupply = "27647235749155415536952630" - var initialSupplyPlusOne = "27647235749155415536952631" - var initialSupplyPlusTwo = "27647235749155415536952632" - var initialSupplyPlusThree = "27647235749155415536952633" - var defaultLastBlock = big.Int{} - - BeforeEach(func() { - blockChain = fakes.NewMockBlockChain() - blockChain.SetLastBlock(&defaultLastBlock) - fetcher = mocks.Fetcher{BlockChain: blockChain} - fetcher.SetSupply(initialSupply) - repository = mocks.TotalSupplyRepository{} - repository.SetMissingBlocks([]int64{config.FirstBlock}) - //setting the mock repository to return the first block as the missing blocks - - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &repository, - } - transformer.SetConfiguration(config) - }) - - It("fetches and persists the total supply of a token for a single block", func() { - err := transformer.Execute() - Expect(err).NotTo(HaveOccurred()) - - Expect(len(fetcher.FetchedBlocks)).To(Equal(1)) - Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock)) - Expect(fetcher.Abi).To(Equal(config.Abi)) - Expect(fetcher.ContractAddress).To(Equal(config.Address)) - - Expect(repository.StartingBlock).To(Equal(config.FirstBlock)) - Expect(repository.EndingBlock).To(Equal(config.LastBlock)) - Expect(len(repository.TotalSuppliesCreated)).To(Equal(1)) - expectedSupply := big.Int{} - expectedSupply.SetString(initialSupply, 10) - expectedSupply.Add(&expectedSupply, big.NewInt(1)) - - Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(expectedSupply.String())) - }) - - It("retrieves the total supply for every missing block", func() { - missingBlocks := []int64{ - config.FirstBlock, - config.FirstBlock + 1, - config.FirstBlock + 2, - } - repository.SetMissingBlocks(missingBlocks) - transformer.Execute() - - Expect(len(fetcher.FetchedBlocks)).To(Equal(3)) - Expect(fetcher.FetchedBlocks).To(ConsistOf(config.FirstBlock, config.FirstBlock+1, config.FirstBlock+2)) - Expect(fetcher.Abi).To(Equal(config.Abi)) - Expect(fetcher.ContractAddress).To(Equal(config.Address)) - - Expect(len(repository.TotalSuppliesCreated)).To(Equal(3)) - Expect(repository.TotalSuppliesCreated[0].Value).To(Equal(initialSupplyPlusOne)) - Expect(repository.TotalSuppliesCreated[1].Value).To(Equal(initialSupplyPlusTwo)) - Expect(repository.TotalSuppliesCreated[2].Value).To(Equal(initialSupplyPlusThree)) - }) - - It("uses the set contract configuration", func() { - repository.SetMissingBlocks([]int64{testContractConfig.FirstBlock}) - transformer.SetConfiguration(testContractConfig) - err := transformer.Execute() - Expect(err).NotTo(HaveOccurred()) - - Expect(fetcher.FetchedBlocks).To(ConsistOf(testContractConfig.FirstBlock)) - Expect(fetcher.Abi).To(Equal(testContractConfig.Abi)) - Expect(fetcher.ContractAddress).To(Equal(testContractConfig.Address)) - - Expect(repository.StartingBlock).To(Equal(testContractConfig.FirstBlock)) - Expect(repository.EndingBlock).To(Equal(testContractConfig.LastBlock)) - Expect(len(repository.TotalSuppliesCreated)).To(Equal(1)) - expectedTokenSupply := every_block.TokenSupply{ - Value: initialSupplyPlusOne, - TokenAddress: testContractConfig.Address, - BlockNumber: testContractConfig.FirstBlock, - } - Expect(repository.TotalSuppliesCreated[0]).To(Equal(expectedTokenSupply)) - }) - - It("uses the most recent block if the Config.LastBlock is -1", func() { - testContractConfig.LastBlock = -1 - transformer.SetConfiguration(testContractConfig) - - randomBlockNumber := rand.Int63() - numberToString := strconv.FormatInt(randomBlockNumber, 10) - mostRecentBlock := big.Int{} - mostRecentBlock.SetString(numberToString, 10) - - blockChain.SetLastBlock(&mostRecentBlock) - - err := transformer.Execute() - Expect(err).NotTo(HaveOccurred()) - - Expect(repository.EndingBlock).To(Equal(randomBlockNumber)) - }) - - It("returns an error if the call to get missing blocks fails", func() { - failureRepository := mocks.FailureRepository{} - failureRepository.SetMissingBlocksFail(true) - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &failureRepository, - } - err := transformer.Execute() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - Expect(err.Error()).To(ContainSubstring("fetching missing blocks")) - }) - - It("returns an error if the call to the blockChain fails", func() { - failureBlockchain := fakes.NewMockBlockChain() - failureBlockchain.SetLastBlock(&defaultLastBlock) - failureBlockchain.SetFetchContractDataErr(fakes.FakeError) - fetcher := every_block.NewFetcher(failureBlockchain) - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &repository, - } - err := transformer.Execute() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - Expect(err.Error()).To(ContainSubstring("supply")) - }) - - It("returns an error if the call to save the token_supply fails", func() { - failureRepository := mocks.FailureRepository{} - failureRepository.SetMissingBlocks([]int64{config.FirstBlock}) - failureRepository.SetCreateFail(true) - - transformer = every_block.Transformer{ - Fetcher: &fetcher, - Repository: &failureRepository, - } - err := transformer.Execute() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error())) - Expect(err.Error()).To(ContainSubstring("supply")) - }) -}) diff --git a/examples/erc20_watcher/every_block/transformers.go b/examples/erc20_watcher/every_block/transformers.go deleted file mode 100644 index f54b3528..00000000 --- a/examples/erc20_watcher/every_block/transformers.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018 Vulcanize -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package every_block - -import ( - "github.com/vulcanize/vulcanizedb/examples/erc20_watcher" - "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" -) - -func TransformerInitializers() []shared.TransformerInitializer { - config := erc20_watcher.DaiConfig - initializer := TokenSupplyTransformerInitializer{config} - return []shared.TransformerInitializer{ - initializer.NewTokenSupplyTransformer, - } -} diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index 1e3c47e9..8a8ded13 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -1,6 +1,8 @@ package shared import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -8,21 +10,82 @@ import ( type Watcher struct { Transformers []shared.Transformer - DB postgres.DB - Blockchain core.BlockChain + DB *postgres.DB + Fetcher shared.LogFetcher + Chunker shared.Chunker + Addresses []common.Address + Topics []common.Hash } -func (watcher *Watcher) AddTransformers(us []shared.TransformerInitializer) { - for _, transformerInitializer := range us { - transformer := transformerInitializer(&watcher.DB, watcher.Blockchain) - watcher.Transformers = append(watcher.Transformers, transformer) +func NewWatcher(db *postgres.DB, bc core.BlockChain) Watcher { + chunker := shared.NewLogChunker() + fetcher := shared.NewFetcher(bc) + return Watcher{ + DB: db, + Fetcher: fetcher, + Chunker: chunker, } } +// Adds transformers to the watcher and updates the chunker, so that it will consider the new transformers. +func (watcher *Watcher) AddTransformers(initializers []shared.TransformerInitializer) { + var contractAddresses []common.Address + var topic0s []common.Hash + var configs []shared.TransformerConfig + + for _, initializer := range initializers { + transformer := initializer(watcher.DB) + watcher.Transformers = append(watcher.Transformers, transformer) + + config := transformer.GetConfig() + configs = append(configs, config) + + addresses := shared.HexStringsToAddresses(config.ContractAddresses) + contractAddresses = append(contractAddresses, addresses...) + topic0s = append(topic0s, common.HexToHash(config.Topic)) + } + + watcher.Addresses = append(watcher.Addresses, contractAddresses...) + watcher.Topics = append(watcher.Topics, topic0s...) + watcher.Chunker.AddConfigs(configs) +} + func (watcher *Watcher) Execute() error { - var err error - for _, transformer := range watcher.Transformers { - err = transformer.Execute() + checkedColumnNames, err := shared.GetCheckedColumnNames(watcher.DB) + if err != nil { + return err + } + notCheckedSQL := shared.CreateNotCheckedSQL(checkedColumnNames) + + // TODO Handle start and end numbers in transformers + missingHeaders, err := shared.MissingHeaders(0, -1, watcher.DB, notCheckedSQL) + if err != nil { + log.Error("Fetching of missing headers failed in watcher!") + return err + } + + for _, header := range missingHeaders { + // TODO Extend FetchLogs for doing several blocks at a time + logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header) + if err != nil { + // TODO Handle fetch error in watcher + log.Error("Error while fetching logs for header %v in watcher", header.Id) + return err + } + + chunkedLogs := watcher.Chunker.ChunkLogs(logs) + + // Can't quit early and mark as checked if there are no logs. If we are running continuousLogSync, + // not all logs we're interested in might have been fetched. + for _, transformer := range watcher.Transformers { + transformerName := transformer.GetConfig().TransformerName + logChunk := chunkedLogs[transformerName] + err = transformer.Execute(logChunk, header) + if err != nil { + log.Error("%v transformer failed to execute in watcher: %v", transformerName, err) + return err + } + } } return err } diff --git a/libraries/shared/watcher_test.go b/libraries/shared/watcher_test.go index d9b1f55c..c93c6336 100644 --- a/libraries/shared/watcher_test.go +++ b/libraries/shared/watcher_test.go @@ -2,69 +2,162 @@ package shared_test import ( "errors" - + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" shared2 "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data/mocks" + "github.com/vulcanize/vulcanizedb/test_config" ) -type MockTransformer struct { - executeWasCalled bool - executeError error -} - -func (mh *MockTransformer) Execute() error { - if mh.executeError != nil { - return mh.executeError - } - mh.executeWasCalled = true - return nil -} - -func fakeTransformerInitializer(db *postgres.DB, blockchain core.BlockChain) shared2.Transformer { - return &MockTransformer{} -} - var _ = Describe("Watcher", func() { - It("Adds transformers", func() { - watcher := shared.Watcher{} + It("initialises correctly", func() { + db := test_config.NewTestDB(core.Node{ID: "testNode"}) + bc := fakes.NewMockBlockChain() - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) + watcher := shared.NewWatcher(db, bc) + + Expect(watcher.DB).To(Equal(db)) + Expect(watcher.Fetcher).NotTo(BeNil()) + Expect(watcher.Chunker).NotTo(BeNil()) + }) + + It("adds transformers", func() { + watcher := shared.NewWatcher(nil, nil) + fakeTransformer := &mocks.MockTransformer{} + fakeTransformer.SetTransformerConfig(mocks.FakeTransformerConfig) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer.FakeTransformerInitializer}) Expect(len(watcher.Transformers)).To(Equal(1)) - Expect(watcher.Transformers).To(ConsistOf(&MockTransformer{})) + Expect(watcher.Transformers).To(ConsistOf(fakeTransformer)) + Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic")})) + Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress")})) }) - It("Adds transformers from multiple sources", func() { - watcher := shared.Watcher{} + It("adds transformers from multiple sources", func() { + watcher := shared.NewWatcher(nil, nil) + fakeTransformer1 := &mocks.MockTransformer{} + fakeTransformer1.SetTransformerConfig(mocks.FakeTransformerConfig) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) - watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformerInitializer}) + fakeTransformer2 := &mocks.MockTransformer{} + fakeTransformer2.SetTransformerConfig(mocks.FakeTransformerConfig) + + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer1.FakeTransformerInitializer}) + watcher.AddTransformers([]shared2.TransformerInitializer{fakeTransformer2.FakeTransformerInitializer}) Expect(len(watcher.Transformers)).To(Equal(2)) + Expect(watcher.Topics).To(Equal([]common.Hash{common.HexToHash("FakeTopic"), + common.HexToHash("FakeTopic")})) + Expect(watcher.Addresses).To(Equal([]common.Address{common.HexToAddress("FakeAddress"), + common.HexToAddress("FakeAddress")})) }) - It("Executes each transformer", func() { - watcher := shared.Watcher{} - fakeTransformer := &MockTransformer{} - watcher.Transformers = []shared2.Transformer{fakeTransformer} + Describe("with missing headers", func() { + var ( + db *postgres.DB + watcher shared.Watcher + mockBlockChain fakes.MockBlockChain + fakeTransformer *mocks.MockTransformer + headerRepository repositories.HeaderRepository + repository mocks.MockWatcherRepository + ) - watcher.Execute() + BeforeEach(func() { + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + mockBlockChain = fakes.MockBlockChain{} + headerRepository = repositories.NewHeaderRepository(db) + _, err := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) + Expect(err).NotTo(HaveOccurred()) - Expect(fakeTransformer.executeWasCalled).To(BeTrue()) - }) + repository = mocks.MockWatcherRepository{} + watcher = shared.NewWatcher(db, &mockBlockChain) + }) - It("Returns an error if transformer returns an error", func() { - watcher := shared.Watcher{} - fakeTransformer := &MockTransformer{executeError: errors.New("Something bad happened")} - watcher.Transformers = []shared2.Transformer{fakeTransformer} + It("executes each transformer", func() { + fakeTransformer = &mocks.MockTransformer{} + watcher.Transformers = []shared2.Transformer{fakeTransformer} + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) - err := watcher.Execute() + err := watcher.Execute() - Expect(err).To(HaveOccurred()) - Expect(fakeTransformer.executeWasCalled).To(BeFalse()) + Expect(err).NotTo(HaveOccurred()) + Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue()) + }) + + It("returns an error if transformer returns an error", func() { + fakeTransformer = &mocks.MockTransformer{ExecuteError: errors.New("Something bad happened")} + watcher.Transformers = []shared2.Transformer{fakeTransformer} + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + + err := watcher.Execute() + + Expect(err).To(HaveOccurred()) + Expect(fakeTransformer.ExecuteWasCalled).To(BeFalse()) + }) + + It("passes only relevant logs to each transformer", func() { + transformerA := &mocks.MockTransformer{} + transformerB := &mocks.MockTransformer{} + + configA := shared2.TransformerConfig{TransformerName: "transformerA", + ContractAddresses: []string{"0x000000000000000000000000000000000000000A"}, + Topic: "0xA"} + configB := shared2.TransformerConfig{TransformerName: "transformerB", + ContractAddresses: []string{"0x000000000000000000000000000000000000000b"}, + Topic: "0xB"} + + transformerA.SetTransformerConfig(configA) + transformerB.SetTransformerConfig(configB) + + logA := types.Log{Address: common.HexToAddress("0xA"), + Topics: []common.Hash{common.HexToHash("0xA")}} + logB := types.Log{Address: common.HexToAddress("0xB"), + Topics: []common.Hash{common.HexToHash("0xB")}} + mockBlockChain.SetGetEthLogsWithCustomQueryReturnLogs([]types.Log{logA, logB}) + + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + watcher = shared.NewWatcher(db, &mockBlockChain) + watcher.AddTransformers([]shared2.TransformerInitializer{ + transformerA.FakeTransformerInitializer, transformerB.FakeTransformerInitializer}) + + err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(transformerA.PassedLogs).To(Equal([]types.Log{logA})) + Expect(transformerB.PassedLogs).To(Equal([]types.Log{logB})) + }) + + Describe("uses the LogFetcher correctly:", func() { + BeforeEach(func() { + repository.SetMissingHeaders([]core.Header{fakes.FakeHeader}) + }) + + It("fetches logs", func() { + err := watcher.Execute() + Expect(err).NotTo(HaveOccurred()) + + fakeHash := common.HexToHash(fakes.FakeHeader.Hash) + mockBlockChain.AssertGetEthLogsWithCustomQueryCalledWith(ethereum.FilterQuery{ + BlockHash: &fakeHash, + Addresses: nil, + Topics: [][]common.Hash{nil}, + }) + }) + + It("propagates log fetcher errors", func() { + fetcherError := errors.New("FetcherError") + mockBlockChain.SetGetEthLogsWithCustomQueryErr(fetcherError) + + err := watcher.Execute() + Expect(err).To(MatchError(fetcherError)) + }) + }) }) }) diff --git a/pkg/datastore/postgres/repositories/header_repository.go b/pkg/datastore/postgres/repositories/header_repository.go index ccc99198..207afc3b 100644 --- a/pkg/datastore/postgres/repositories/header_repository.go +++ b/pkg/datastore/postgres/repositories/header_repository.go @@ -3,7 +3,6 @@ package repositories import ( "database/sql" "errors" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) diff --git a/pkg/transformers/DOCUMENTATION.md b/pkg/transformers/DOCUMENTATION.md index ed3e8a4c..de85a455 100644 --- a/pkg/transformers/DOCUMENTATION.md +++ b/pkg/transformers/DOCUMENTATION.md @@ -25,7 +25,7 @@ The transformer process for each of these different log types is the same, excep ## Creating a Transformer 1. Pull an example event (from kovan / ganache etc.) -1. Add event sig to [`constants.go`](./shared/constants.go) +1. Add event & method sig, contract address, `checked_headers` column name, and label to relevant files in [`constants`](./shared/constants) 1. Write a test for the event sig in [`event_signature_generator_test.go`](./shared/event_signature_generator_test.go) 1. Create DB table (using [`create_migration`](../../scripts/create_migration)) 1. Create columns in `checked_headers` in the _same_ migration @@ -36,23 +36,23 @@ The transformer process for each of these different log types is the same, excep 1. Write repository + repository tests 1. Create converter + repository mocks 1. Create an config object [`shared.TransformerConfig`](./shared/transformer.go) in `config.go` -1. Create transformer + transformer tests -1. Wire up transformer in [`transformers.go`](./transformers.go) +1. Wire up transformer in [`transformers.go`](./transformers.go), remembering to add it to `TransformerInitializers()` 1. Wire up transformer in [`continuousLogSync.go`](../../cmd/continuousLogSync.go) 1. Manually trigger an event and check that it gets persisted to postgres +1. Create an integration test for the shiny new transformer in [`integration_tests`](./integration_tests) **Fetching Logs** 1. Generate an example raw log event, by either: - - Pulling the log directly from the Kovan deployment ([constants.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants.go)). + - Pulling the log directly from the Kovan deployment ([address.go](https://github.com/8thlight/maker-vulcanizedb/blob/master/pkg/transformers/shared/constants/address.go)). - Deploying the contract to a local chain and emiting the event manually. 1. Fetch the logs from the chain based on the example event's topic zero: - - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants.go`](./shared/constants.go). - - Most transformers use `shared.LogFetcher` to fetch all logs that match the given topic zero for that log event. - - Since there are multiple price feed contract address that all use the same `LogValue` event, we have a special implementation of a fetcher specifically for price feeds that can query using all of the contract addresses at once, thus only needing to make one call to the blockchain. + - The topic zero is based on the keccak-256 hash of the log event's method signature. These are located in [`pkg/transformers/shared/constants/signature.go`](./shared/constants/signature.go). + - Fetching is done in batch from the [`watcher`](../../libraries/shared/watcher.go). + - The logs are then chunked up by the [`chunker`](./shared/log_chunker.go) before being delegated to each transformer. **Coverting logs** @@ -105,8 +105,7 @@ The transformer process for each of these different log types is the same, excep - The `checked_headers` table allows us to keep track of which headers have been checked for a given log type. - To create a new migration file: `./scripts/create_migration create_flop_kick` - See `db/migrations/1536942529_create_flop_kick.up.sql`. - - The specific log event tables are all created in the `maker` - schema. + - The specific log event tables are all created in the `maker` schema. - There is a one-many association between `headers` and the log event tables. This is so that if a header is removed due to a reorg, the associated log event records are also removed. - To run the migrations: `make migrate HOST=local_host PORT=5432 NAME=vulcanize_private` @@ -120,7 +119,7 @@ The transformer process for each of these different log types is the same, excep **Wire each component up in the transformer** -- We use a TransformerInitializer struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. The TransformerInitializer interface is defined in `pkg/transformers/shared/transformer.go`. +- We use a [`TransformerInitializer`](./shared/transformer.go) struct for each transformer so that we can inject ethRPC and postgresDB connections as well as configuration data (including the contract address, block range, etc.) into the transformer. - See any of `pkg/transformers/flop_kick/transformer.go` - All of the transformers are then initialized in `pkg/transformers/transformers.go` with their configuration. - The transformers can be executed by using the `continuousLogSync` command, which can be configured to run specific transformers or all transformers. diff --git a/pkg/transformers/bite/repository.go b/pkg/transformers/bite/repository.go index 54cdad84..1e3d860b 100644 --- a/pkg/transformers/bite/repository.go +++ b/pkg/transformers/bite/repository.go @@ -16,7 +16,6 @@ package bite import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -65,7 +64,3 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er func (repository BiteRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.BiteChecked) } - -func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.BiteChecked) -} diff --git a/pkg/transformers/bite/repository_test.go b/pkg/transformers/bite/repository_test.go index 480fe165..c8d48f68 100644 --- a/pkg/transformers/bite/repository_test.go +++ b/pkg/transformers/bite/repository_test.go @@ -86,14 +86,4 @@ var _ = Describe("Bite repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &biteRepository, - RepositoryTwo: &bite.BiteRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/cat_file/chop_lump/repository.go b/pkg/transformers/cat_file/chop_lump/repository.go index 52ef1446..9dff41f7 100644 --- a/pkg/transformers/cat_file/chop_lump/repository.go +++ b/pkg/transformers/cat_file/chop_lump/repository.go @@ -16,7 +16,6 @@ package chop_lump import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository CatFileChopLumpRepository) MarkHeaderChecked(headerID int64) er return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileChopLumpChecked) } -func (repository CatFileChopLumpRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileChopLumpChecked) -} - func (repository *CatFileChopLumpRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/chop_lump/repository_test.go b/pkg/transformers/cat_file/chop_lump/repository_test.go index ec10826a..41041f49 100644 --- a/pkg/transformers/cat_file/chop_lump/repository_test.go +++ b/pkg/transformers/cat_file/chop_lump/repository_test.go @@ -98,13 +98,4 @@ var _ = Describe("Cat file chop lump repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &catFileRepository, - RepositoryTwo: &chop_lump.CatFileChopLumpRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/cat_file/flip/repository.go b/pkg/transformers/cat_file/flip/repository.go index 8f59c20f..a07ca207 100644 --- a/pkg/transformers/cat_file/flip/repository.go +++ b/pkg/transformers/cat_file/flip/repository.go @@ -16,7 +16,6 @@ package flip import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository CatFileFlipRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFileFlipChecked) } -func (repository CatFileFlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFileFlipChecked) -} - func (repository *CatFileFlipRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/flip/repository_test.go b/pkg/transformers/cat_file/flip/repository_test.go index c50e3a97..8dfe6027 100644 --- a/pkg/transformers/cat_file/flip/repository_test.go +++ b/pkg/transformers/cat_file/flip/repository_test.go @@ -83,12 +83,4 @@ var _ = Describe("Cat file flip repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &catFileFlipRepository, - RepositoryTwo: &flip.CatFileFlipRepository{}, - } - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/cat_file/pit_vow/repository.go b/pkg/transformers/cat_file/pit_vow/repository.go index 6aa7a108..861dc2a8 100644 --- a/pkg/transformers/cat_file/pit_vow/repository.go +++ b/pkg/transformers/cat_file/pit_vow/repository.go @@ -16,7 +16,6 @@ package pit_vow import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository CatFilePitVowRepository) MarkHeaderChecked(headerID int64) erro return shared.MarkHeaderChecked(headerID, repository.db, constants.CatFilePitVowChecked) } -func (repository CatFilePitVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.CatFilePitVowChecked) -} - func (repository *CatFilePitVowRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/cat_file/pit_vow/repository_test.go b/pkg/transformers/cat_file/pit_vow/repository_test.go index 5f97cfe1..c7d31167 100644 --- a/pkg/transformers/cat_file/pit_vow/repository_test.go +++ b/pkg/transformers/cat_file/pit_vow/repository_test.go @@ -81,13 +81,4 @@ var _ = Describe("Cat file pit vow repository", func() { } shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &catFilePitVowRepository, - RepositoryTwo: &pit_vow.CatFilePitVowRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/deal/repository.go b/pkg/transformers/deal/repository.go index 84a40325..06f23230 100644 --- a/pkg/transformers/deal/repository.go +++ b/pkg/transformers/deal/repository.go @@ -16,7 +16,6 @@ package deal import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository DealRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.DealChecked) } -func (repository DealRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DealChecked) -} - func (repository *DealRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/deal/repository_test.go b/pkg/transformers/deal/repository_test.go index 5a58c4ca..2e3f02b0 100644 --- a/pkg/transformers/deal/repository_test.go +++ b/pkg/transformers/deal/repository_test.go @@ -84,13 +84,4 @@ var _ = Describe("Deal Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dealRepository, - RepositoryTwo: &deal.DealRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/dent/repository.go b/pkg/transformers/dent/repository.go index e237fb54..0ff5370b 100644 --- a/pkg/transformers/dent/repository.go +++ b/pkg/transformers/dent/repository.go @@ -16,7 +16,6 @@ package dent import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -67,10 +66,6 @@ func (repository DentRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.DentChecked) } -func (repository DentRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DentChecked) -} - func (repository *DentRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/dent/repository_test.go b/pkg/transformers/dent/repository_test.go index bad19213..b667228a 100644 --- a/pkg/transformers/dent/repository_test.go +++ b/pkg/transformers/dent/repository_test.go @@ -92,13 +92,4 @@ var _ = Describe("Dent Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dentRepository, - RepositoryTwo: &dent.DentRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_drip/repository.go b/pkg/transformers/drip_drip/repository.go index 95780c74..edb1208a 100644 --- a/pkg/transformers/drip_drip/repository.go +++ b/pkg/transformers/drip_drip/repository.go @@ -16,7 +16,6 @@ package drip_drip import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository DripDripRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.DripDripChecked) } -func (repository DripDripRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripDripChecked) -} - func (repository *DripDripRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_drip/repository_test.go b/pkg/transformers/drip_drip/repository_test.go index d66b9fd0..50646855 100644 --- a/pkg/transformers/drip_drip/repository_test.go +++ b/pkg/transformers/drip_drip/repository_test.go @@ -80,13 +80,4 @@ var _ = Describe("Drip drip repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripDripRepository, - RepositoryTwo: &drip_drip.DripDripRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_file/ilk/repository.go b/pkg/transformers/drip_file/ilk/repository.go index 8c602ef9..117a28fd 100644 --- a/pkg/transformers/drip_file/ilk/repository.go +++ b/pkg/transformers/drip_file/ilk/repository.go @@ -16,7 +16,6 @@ package ilk import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository DripFileIlkRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileIlkChecked) } -func (repository DripFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileIlkChecked) -} - func (repository *DripFileIlkRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/ilk/repository_test.go b/pkg/transformers/drip_file/ilk/repository_test.go index 6853f291..8dbada31 100644 --- a/pkg/transformers/drip_file/ilk/repository_test.go +++ b/pkg/transformers/drip_file/ilk/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Drip file ilk repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripFileIlkRepository, - RepositoryTwo: &ilk.DripFileIlkRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_file/repo/repository.go b/pkg/transformers/drip_file/repo/repository.go index 592e92e6..dc9589a7 100644 --- a/pkg/transformers/drip_file/repo/repository.go +++ b/pkg/transformers/drip_file/repo/repository.go @@ -16,7 +16,6 @@ package repo import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository DripFileRepoRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileRepoChecked) } -func (repository DripFileRepoRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileRepoChecked) -} - func (repository *DripFileRepoRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/repo/repository_test.go b/pkg/transformers/drip_file/repo/repository_test.go index 8e297a3c..e8614b61 100644 --- a/pkg/transformers/drip_file/repo/repository_test.go +++ b/pkg/transformers/drip_file/repo/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Drip file repo repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripFileRepoRepository, - RepositoryTwo: &repo.DripFileRepoRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/drip_file/vow/repository.go b/pkg/transformers/drip_file/vow/repository.go index 35b089d0..aa23bea8 100644 --- a/pkg/transformers/drip_file/vow/repository.go +++ b/pkg/transformers/drip_file/vow/repository.go @@ -16,7 +16,6 @@ package vow import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -63,10 +62,6 @@ func (repository DripFileVowRepository) MarkHeaderChecked(headerID int64) error return shared.MarkHeaderChecked(headerID, repository.db, constants.DripFileVowChecked) } -func (repository DripFileVowRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.DripFileVowChecked) -} - func (repository *DripFileVowRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/drip_file/vow/repository_test.go b/pkg/transformers/drip_file/vow/repository_test.go index 9ad62a49..71e8cd07 100644 --- a/pkg/transformers/drip_file/vow/repository_test.go +++ b/pkg/transformers/drip_file/vow/repository_test.go @@ -74,15 +74,6 @@ var _ = Describe("Drip file vow repository", func() { }) Describe("MarkHeaderChecked", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &dripFileVowRepository, - RepositoryTwo: &vow.DripFileVowRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - - Describe("MissingHeaders", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.DripFileVowChecked, Repository: &dripFileVowRepository, diff --git a/pkg/transformers/factories/log_note_transformer.go b/pkg/transformers/factories/log_note_transformer.go index b237cd87..6ae63f4b 100644 --- a/pkg/transformers/factories/log_note_transformer.go +++ b/pkg/transformers/factories/log_note_transformer.go @@ -15,10 +15,9 @@ package factories import ( + "github.com/ethereum/go-ethereum/core/types" log "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -28,58 +27,44 @@ type LogNoteTransformer struct { Config shared.TransformerConfig Converter LogNoteConverter Repository Repository - Fetcher shared.SettableLogFetcher } -func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { +func (transformer LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) shared.Transformer { transformer.Repository.SetDB(db) - transformer.Fetcher.SetBC(bc) return transformer } -func (transformer LogNoteTransformer) Execute() error { +func (transformer LogNoteTransformer) Execute(logs []types.Log, header core.Header) error { transformerName := transformer.Config.TransformerName - missingHeaders, err := transformer.Repository.MissingHeaders(transformer.Config.StartingBlockNumber, transformer.Config.EndingBlockNumber) + + // No matching logs, mark the header as checked for this type of logs + if len(logs) < 1 { + err := transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + log.Printf("Error marking header as checked in %v: %v", transformerName, err) + return err + } + return nil + } + + models, err := transformer.Converter.ToModels(logs) if err != nil { - log.Printf("Error fetching mising headers in %v transformer: %v", transformerName, err) + log.Printf("Error converting logs in %v: %v", transformerName, err) return err } - // Grab event signature from transformer config - // (Double-array structure required for go-ethereum FilterQuery) - var topic = [][]common.Hash{{common.HexToHash(transformer.Config.Topic)}} - - log.Printf("Fetching %v event logs for %d headers", transformerName, len(missingHeaders)) - for _, header := range missingHeaders { - // Fetch the missing logs for a given header - matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header) - if err != nil { - log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) - return err - } - - // No matching logs, mark the header as checked for this type of logs - if len(matchingLogs) < 1 { - err := transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - log.Printf("Error marking header as checked in %v: %v", transformerName, err) - return err - } - // Continue with the next header; nothing to persist - continue - } - - models, err := transformer.Converter.ToModels(matchingLogs) - if err != nil { - log.Printf("Error converting logs in %v: %v", transformerName, err) - return err - } - - err = transformer.Repository.Create(header.Id, models) - if err != nil { - log.Printf("Error persisting %v record: %v", transformerName, err) - return err - } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + log.Printf("Error persisting %v record: %v", transformerName, err) + return err } return nil } + +func (transformer LogNoteTransformer) GetName() string { + return transformer.Config.TransformerName +} + +func (transformer LogNoteTransformer) GetConfig() shared.TransformerConfig { + return transformer.Config +} diff --git a/pkg/transformers/factories/log_note_transformer_test.go b/pkg/transformers/factories/log_note_transformer_test.go index d6eb2e24..678c9879 100644 --- a/pkg/transformers/factories/log_note_transformer_test.go +++ b/pkg/transformers/factories/log_note_transformer_test.go @@ -15,7 +15,7 @@ package factories_test import ( - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -31,9 +31,7 @@ var _ = Describe("LogNoteTransformer", func() { var ( repository mocks.MockRepository converter mocks.MockLogNoteConverter - fetcher mocks.MockLogFetcher headerOne core.Header - headerTwo core.Header transformer shared.Transformer model test_data.GenericModel config = test_data.GenericTestConfig @@ -43,75 +41,28 @@ var _ = Describe("LogNoteTransformer", func() { BeforeEach(func() { repository = mocks.MockRepository{} converter = mocks.MockLogNoteConverter{} - fetcher = mocks.MockLogFetcher{} transformer = factories.LogNoteTransformer{ Config: config, Converter: &converter, Repository: &repository, - Fetcher: &fetcher, - }.NewLogNoteTransformer(nil, nil) + }.NewLogNoteTransformer(nil) headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} - headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} }) - It("sets the blockchain and database", func() { - Expect(fetcher.SetBcCalled).To(BeTrue()) + It("sets the database", func() { Expect(repository.SetDbCalled).To(BeTrue()) }) - It("gets missing headers for block numbers specified in config", func() { - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) - }) - - It("returns error if repository returns error for missing headers", func() { - repository.SetMissingHeadersError(fakes.FakeError) - - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("fetches logs for missing headers", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) - expectedTopic := common.HexToHash(config.Topic) - Expect(fetcher.FetchedTopics).To(Equal([][]common.Hash{{expectedTopic}})) - }) - - It("returns error if fetcher returns error", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetcherError(fakes.FakeError) - - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("marks header checked if no logs returned", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - - err := transformer.Execute() + It("marks header checked if no logs are provided", func() { + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.ToModelsCalledCounter).To(Equal(0)) @@ -119,30 +70,23 @@ var _ = Describe("LogNoteTransformer", func() { }) It("does not call repository.MarkCheckedHeader when there are logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) - - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedNotCalled() }) It("returns error if marking header checked returns err", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetMarkHeaderCheckedError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts matching logs to models", func() { - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) - - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.PassedLogs).To(Equal(logs)) @@ -150,20 +94,16 @@ var _ = Describe("LogNoteTransformer", func() { It("returns error if converter returns error", func() { converter.SetConverterError(fakes.FakeError) - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("persists the model", func() { - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) converter.SetReturnModels([]interface{}{model}) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) @@ -171,11 +111,9 @@ var _ = Describe("LogNoteTransformer", func() { }) It("returns error if repository returns error for create", func() { - fetcher.SetFetchedLogs(logs) - repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetCreateError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/factories/repository.go b/pkg/transformers/factories/repository.go index 015fe49f..0ddc9274 100644 --- a/pkg/transformers/factories/repository.go +++ b/pkg/transformers/factories/repository.go @@ -15,13 +15,11 @@ package factories import ( - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) type Repository interface { Create(headerID int64, models []interface{}) error MarkHeaderChecked(headerID int64) error - MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) SetDB(db *postgres.DB) } diff --git a/pkg/transformers/factories/transformer.go b/pkg/transformers/factories/transformer.go index 1fe13236..5b164814 100644 --- a/pkg/transformers/factories/transformer.go +++ b/pkg/transformers/factories/transformer.go @@ -15,10 +15,9 @@ package factories import ( + "github.com/ethereum/go-ethereum/core/types" log "github.com/sirupsen/logrus" - "github.com/ethereum/go-ethereum/common" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -28,61 +27,51 @@ type Transformer struct { Config shared.TransformerConfig Converter Converter Repository Repository - Fetcher shared.SettableLogFetcher } -func (transformer Transformer) NewTransformer(db *postgres.DB, bc core.BlockChain) shared.Transformer { +func (transformer Transformer) NewTransformer(db *postgres.DB) shared.Transformer { transformer.Repository.SetDB(db) - transformer.Fetcher.SetBC(bc) return transformer } -func (transformer Transformer) Execute() error { +func (transformer Transformer) Execute(logs []types.Log, header core.Header) error { transformerName := transformer.Config.TransformerName config := transformer.Config - topics := [][]common.Hash{{common.HexToHash(config.Topic)}} - missingHeaders, err := transformer.Repository.MissingHeaders(config.StartingBlockNumber, config.EndingBlockNumber) + + if len(logs) < 1 { + err := transformer.Repository.MarkHeaderChecked(header.Id) + if err != nil { + log.Printf("Error marking header as checked in %v: %v", transformerName, err) + return err + } + return nil + } + + entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs) if err != nil { - log.Printf("Error fetching missing headers in %v transformer: %v \n", transformerName, err) + log.Printf("Error converting logs to entities in %v: %v", transformerName, err) return err } - log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders)) - for _, header := range missingHeaders { - logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header) - if err != nil { - log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) - return err - } - if len(logs) < 1 { - err = transformer.Repository.MarkHeaderChecked(header.Id) - if err != nil { - log.Printf("Error marking header as checked in %v: %v", transformerName, err) - return err - } - - continue - } - - entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs) - if err != nil { - log.Printf("Error converting logs to entities in %v: %v", transformerName, err) - return err - } - - models, err := transformer.Converter.ToModels(entities) - if err != nil { - log.Printf("Error converting entities to models in %v: %v", transformerName, err) - return err - } - - err = transformer.Repository.Create(header.Id, models) - if err != nil { - log.Printf("Error persisting %v record: %v", transformerName, err) - return err - } + models, err := transformer.Converter.ToModels(entities) + if err != nil { + log.Printf("Error converting entities to models in %v: %v", transformerName, err) + return err + } + err = transformer.Repository.Create(header.Id, models) + if err != nil { + log.Printf("Error persisting %v record: %v", transformerName, err) + return err } return nil } + +func (transformer Transformer) GetName() string { + return transformer.Config.TransformerName +} + +func (transformer Transformer) GetConfig() shared.TransformerConfig { + return transformer.Config +} diff --git a/pkg/transformers/factories/transformer_test.go b/pkg/transformers/factories/transformer_test.go index 929b7f8d..32d85880 100644 --- a/pkg/transformers/factories/transformer_test.go +++ b/pkg/transformers/factories/transformer_test.go @@ -15,7 +15,7 @@ package factories_test import ( - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -30,86 +30,39 @@ import ( var _ = Describe("Transformer", func() { var ( repository mocks.MockRepository - fetcher mocks.MockLogFetcher converter mocks.MockConverter transformer shared.Transformer headerOne core.Header - headerTwo core.Header config = test_data.GenericTestConfig logs = test_data.GenericTestLogs ) BeforeEach(func() { repository = mocks.MockRepository{} - fetcher = mocks.MockLogFetcher{} converter = mocks.MockConverter{} transformer = factories.Transformer{ Repository: &repository, - Fetcher: &fetcher, Converter: &converter, Config: config, - }.NewTransformer(nil, nil) + }.NewTransformer(nil) headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} - headerTwo = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()} }) - It("sets the blockchain and db", func() { - Expect(fetcher.SetBcCalled).To(BeTrue()) + It("sets the db", func() { Expect(repository.SetDbCalled).To(BeTrue()) }) - It("gets missing headers for blocks in the configured range", func() { - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(repository.PassedStartingBlockNumber).To(Equal(config.StartingBlockNumber)) - Expect(repository.PassedEndingBlockNumber).To(Equal(config.EndingBlockNumber)) - }) - - It("returns an error if it fails to get missing headers", func() { - repository.SetMissingHeadersError(fakes.FakeError) - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("fetches eth logs for each missing header", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - expectedTopics := [][]common.Hash{{common.HexToHash(config.Topic)}} - err := transformer.Execute() - - Expect(err).NotTo(HaveOccurred()) - Expect(fetcher.FetchedBlocks).To(Equal([]int64{headerOne.BlockNumber, headerTwo.BlockNumber})) - Expect(fetcher.FetchedTopics).To(Equal(expectedTopics)) - Expect(fetcher.FetchedContractAddresses).To(Equal([][]string{config.ContractAddresses, config.ContractAddresses})) - }) - - It("returns an error if fetching logs fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetcherError(fakes.FakeError) - err := transformer.Execute() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - It("marks header checked if no logs returned", func() { - headerID := int64(123) - repository.SetMissingHeaders([]core.Header{{Id: headerID}}) - - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) - repository.AssertMarkHeaderCheckedCalledWith(headerID) + repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id) }) It("doesn't attempt to convert or persist an empty collection when there are no logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne, headerTwo}) - - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.ToEntitiesCalledCounter).To(Equal(0)) @@ -118,29 +71,23 @@ var _ = Describe("Transformer", func() { }) It("does not call repository.MarkCheckedHeader when there are logs", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) - - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) repository.AssertMarkHeaderCheckedNotCalled() }) It("returns error if marking header checked returns err", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) repository.SetMarkHeaderCheckedError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute([]types.Log{}, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts an eth log to an entity", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.ContractAbi).To(Equal(config.ContractAbi)) @@ -148,45 +95,37 @@ var _ = Describe("Transformer", func() { }) It("returns an error if converter fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.ToEntitiesError = fakes.FakeError - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("converts an entity to a model", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{})) }) It("returns an error if converting to models fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}} converter.ToModelsError = fakes.FakeError - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) }) It("persists the record", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) converter.ModelsToReturn = []interface{}{test_data.GenericModel{}} - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).NotTo(HaveOccurred()) Expect(repository.PassedHeaderID).To(Equal(headerOne.Id)) @@ -194,10 +133,8 @@ var _ = Describe("Transformer", func() { }) It("returns error if persisting the record fails", func() { - repository.SetMissingHeaders([]core.Header{headerOne}) - fetcher.SetFetchedLogs(logs) repository.SetCreateError(fakes.FakeError) - err := transformer.Execute() + err := transformer.Execute(logs, headerOne) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/flap_kick/repository.go b/pkg/transformers/flap_kick/repository.go index 59f5803a..f9f11c7a 100644 --- a/pkg/transformers/flap_kick/repository.go +++ b/pkg/transformers/flap_kick/repository.go @@ -16,7 +16,6 @@ package flap_kick import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -60,10 +59,6 @@ func (repository *FlapKickRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.FlapKickChecked) } -func (repository FlapKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlapKickChecked) -} - func (repository *FlapKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flap_kick/repository_test.go b/pkg/transformers/flap_kick/repository_test.go index 1012548b..2b401cfd 100644 --- a/pkg/transformers/flap_kick/repository_test.go +++ b/pkg/transformers/flap_kick/repository_test.go @@ -87,13 +87,4 @@ var _ = Describe("Flap Kick Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &flapKickRepository, - RepositoryTwo: &flap_kick.FlapKickRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/flip_kick/repository.go b/pkg/transformers/flip_kick/repository.go index 21de65bc..db8a8d01 100644 --- a/pkg/transformers/flip_kick/repository.go +++ b/pkg/transformers/flip_kick/repository.go @@ -17,7 +17,6 @@ package flip_kick import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -60,10 +59,6 @@ func (repository FlipKickRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.FlipKickChecked) } -func (repository FlipKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlipKickChecked) -} - func (repository *FlipKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flip_kick/repository_test.go b/pkg/transformers/flip_kick/repository_test.go index 7ecb5cb0..6befc361 100644 --- a/pkg/transformers/flip_kick/repository_test.go +++ b/pkg/transformers/flip_kick/repository_test.go @@ -87,15 +87,6 @@ var _ = Describe("FlipKick Repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("missing headers", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &flipKickRepository, - RepositoryTwo: &flip_kick.FlipKickRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) func assertDBRecordCount(db *postgres.DB, dbTable string, expectedCount int) { diff --git a/pkg/transformers/flop_kick/repository.go b/pkg/transformers/flop_kick/repository.go index b175caf9..7a302c75 100644 --- a/pkg/transformers/flop_kick/repository.go +++ b/pkg/transformers/flop_kick/repository.go @@ -16,7 +16,6 @@ package flop_kick import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository FlopKickRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.FlopKickChecked) } -func (repository FlopKickRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FlopKickChecked) -} - func (repository *FlopKickRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/flop_kick/repository_test.go b/pkg/transformers/flop_kick/repository_test.go index c916169d..8d5a8351 100644 --- a/pkg/transformers/flop_kick/repository_test.go +++ b/pkg/transformers/flop_kick/repository_test.go @@ -85,13 +85,4 @@ var _ = Describe("FlopRepository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &flop_kick.FlopKickRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/frob/repository.go b/pkg/transformers/frob/repository.go index afe7c4b1..bf6d4d08 100644 --- a/pkg/transformers/frob/repository.go +++ b/pkg/transformers/frob/repository.go @@ -17,7 +17,6 @@ package frob import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -59,10 +58,6 @@ func (repository FrobRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.FrobChecked) } -func (repository FrobRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.FrobChecked) -} - func (repository *FrobRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/frob/repository_test.go b/pkg/transformers/frob/repository_test.go index 5d399edb..36c6036a 100644 --- a/pkg/transformers/frob/repository_test.go +++ b/pkg/transformers/frob/repository_test.go @@ -86,13 +86,4 @@ var _ = Describe("Frob repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &frobRepository, - RepositoryTwo: &frob.FrobRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/integration_tests/bite.go b/pkg/transformers/integration_tests/bite.go index 86c9f7ff..106f7c4f 100644 --- a/pkg/transformers/integration_tests/bite.go +++ b/pkg/transformers/integration_tests/bite.go @@ -45,17 +45,24 @@ var _ = Describe("Bite Transformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.Transformer{ Config: config, Converter: &bite.BiteConverter{}, Repository: &bite.BiteRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []bite.BiteModel diff --git a/pkg/transformers/integration_tests/cat_file.go b/pkg/transformers/integration_tests/cat_file.go index 68983f88..06679e83 100644 --- a/pkg/transformers/integration_tests/cat_file.go +++ b/pkg/transformers/integration_tests/cat_file.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "sort" @@ -39,6 +40,7 @@ var _ = Describe("Cat File transformer", func() { rpcClient client.RpcClient err error ethClient *ethclient.Client + fetcher *shared.Fetcher ) BeforeEach(func() { @@ -48,13 +50,15 @@ var _ = Describe("Cat File transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + fetcher = shared.NewFetcher(blockChain) }) // Cat contract Kovan address: 0x2f34f22a00ee4b7a8f8bbc4eaee1658774c624e0 It("persists a chop lump event", func() { // transaction: 0x98574bfba4d05c3875be10d2376e678d005dbebe9a4520363407508fd21f4014 chopLumpBlockNumber := int64(8762253) - err = persistHeader(db, chopLumpBlockNumber, blockChain) + header, err := persistHeader(db, chopLumpBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := chop_lump.CatFileChopLumpConfig @@ -65,10 +69,16 @@ var _ = Describe("Cat File transformer", func() { Config: config, Converter: &chop_lump.CatFileChopLumpConverter{}, Repository: &chop_lump.CatFileChopLumpRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []chop_lump.CatFileChopLumpModel @@ -92,7 +102,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a flip event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe flipBlockNumber := int64(8751794) - err = persistHeader(db, flipBlockNumber, blockChain) + header, err := persistHeader(db, flipBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := flip.CatFileFlipConfig @@ -103,10 +113,17 @@ var _ = Describe("Cat File transformer", func() { Config: config, Converter: &flip.CatFileFlipConverter{}, Repository: &flip.CatFileFlipRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err := transformer.Execute() + + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flip.CatFileFlipModel @@ -122,7 +139,7 @@ var _ = Describe("Cat File transformer", func() { It("persists a pit vow event", func() { // transaction: 0x44bc18fdb1a5a263db114e7879653304db3e19ceb4e4496f21bc0a76c5faccbe pitVowBlockNumber := int64(8751794) - err = persistHeader(db, pitVowBlockNumber, blockChain) + header, err := persistHeader(db, pitVowBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) config := pit_vow.CatFilePitVowConfig @@ -133,10 +150,16 @@ var _ = Describe("Cat File transformer", func() { Config: config, Converter: &pit_vow.CatFilePitVowConverter{}, Repository: &pit_vow.CatFilePitVowRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err := transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + logs, err := fetcher.FetchLogs( + []common.Address{common.HexToAddress(config.ContractAddresses[0])}, + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []pit_vow.CatFilePitVowModel diff --git a/pkg/transformers/integration_tests/deal.go b/pkg/transformers/integration_tests/deal.go index a43e04cf..b010cced 100644 --- a/pkg/transformers/integration_tests/deal.go +++ b/pkg/transformers/integration_tests/deal.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,8 +30,13 @@ import ( var _ = Describe("Deal transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + initializer factories.LogNoteTransformer + fetcher *shared.Fetcher + addresses []common.Address + topics []common.Hash ) BeforeEach(func() { @@ -40,26 +46,35 @@ var _ = Describe("Deal transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = deal.DealConfig + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &deal.DealConverter{}, + Repository: &deal.DealRepository{}, + } + + fetcher = shared.NewFetcher(blockChain) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} + }) It("persists a flip deal log event", func() { // transaction: 0x05b5eabac2ace136f0f7e0efc61d7d42abe8e8938cc0f04fbf1a6ba545d59e58 flipBlockNumber := int64(8958007) - err := persistHeader(db, flipBlockNumber, blockChain) + header, err := persistHeader(db, flipBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := deal.DealConfig - config.StartingBlockNumber = flipBlockNumber - config.EndingBlockNumber = flipBlockNumber + initializer.Config.StartingBlockNumber = flipBlockNumber + initializer.Config.EndingBlockNumber = flipBlockNumber - initializer := factories.LogNoteTransformer{ - Config: config, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel @@ -77,21 +92,17 @@ var _ = Describe("Deal transformer", func() { It("persists a flap deal log event", func() { flapBlockNumber := int64(9004628) - err := persistHeader(db, flapBlockNumber, blockChain) + header, err := persistHeader(db, flapBlockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := deal.DealConfig - config.StartingBlockNumber = flapBlockNumber - config.EndingBlockNumber = flapBlockNumber + initializer.Config.StartingBlockNumber = flapBlockNumber + initializer.Config.EndingBlockNumber = flapBlockNumber - initializer := factories.LogNoteTransformer{ - Config: config, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []deal.DealModel diff --git a/pkg/transformers/integration_tests/dent.go b/pkg/transformers/integration_tests/dent.go index 47e6c7ac..d1c8050c 100644 --- a/pkg/transformers/integration_tests/dent.go +++ b/pkg/transformers/integration_tests/dent.go @@ -1,6 +1,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -15,8 +16,14 @@ import ( var _ = Describe("Dent transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + fetcher *shared.Fetcher + transformer shared.Transformer + config shared.TransformerConfig + addresses []common.Address + topics []common.Hash + initializer factories.LogNoteTransformer ) BeforeEach(func() { @@ -26,25 +33,32 @@ var _ = Describe("Dent transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = dent.DentConfig + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} + fetcher = shared.NewFetcher(blockChain) + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &dent.DentConverter{}, + Repository: &dent.DentRepository{}, + } }) It("persists a flop dent log event", func() { blockNumber := int64(8955613) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := dent.DentConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - initializer := factories.LogNoteTransformer{ - Config: config, - Converter: &dent.DentConverter{}, - Repository: &dent.DentRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer = initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []dent.DentModel diff --git a/pkg/transformers/integration_tests/drip_drip.go b/pkg/transformers/integration_tests/drip_drip.go index 6203b2dc..783616f0 100644 --- a/pkg/transformers/integration_tests/drip_drip.go +++ b/pkg/transformers/integration_tests/drip_drip.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,32 +28,43 @@ import ( ) var _ = Describe("DripDrip Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) It("transforms DripDrip log events", func() { blockNumber := int64(8934775) config := drip_drip.DripDripConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, Converter: &drip_drip.DripDripConverter{}, Repository: &drip_drip.DripDripRepository{}, - Fetcher: &shared.Fetcher{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) + transformer := initializer.NewLogNoteTransformer(db) - err = transformer.Execute() + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []drip_drip.DripDripModel diff --git a/pkg/transformers/integration_tests/drip_file_vow.go b/pkg/transformers/integration_tests/drip_file_vow.go index 08630370..3f2f065b 100644 --- a/pkg/transformers/integration_tests/drip_file_vow.go +++ b/pkg/transformers/integration_tests/drip_file_vow.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,31 +28,44 @@ import ( ) var _ = Describe("Drip File Vow LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("transforms DripFileVow log events", func() { blockNumber := int64(8762197) config := vow.DripFileVowConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) - Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vow.DripFileVowConverter{}, Repository: &vow.DripFileVowRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vow.DripFileVowModel diff --git a/pkg/transformers/integration_tests/flap_kick.go b/pkg/transformers/integration_tests/flap_kick.go index 6b317f6e..f7ddee53 100644 --- a/pkg/transformers/integration_tests/flap_kick.go +++ b/pkg/transformers/integration_tests/flap_kick.go @@ -15,6 +15,10 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/test_config" "time" . "github.com/onsi/ginkgo" @@ -23,35 +27,46 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/flap_kick" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" - "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("FlapKick Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a FlapKick event from Kovan chain", func() { blockNumber := int64(9002933) config := flap_kick.FlapKickConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) - Expect(err).NotTo(HaveOccurred()) - - initializer := factories.Transformer{ + transformer := factories.Transformer{ Config: config, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + }.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flap_kick.FlapKickModel diff --git a/pkg/transformers/integration_tests/flip_kick.go b/pkg/transformers/integration_tests/flip_kick.go index 17076b47..598542d4 100644 --- a/pkg/transformers/integration_tests/flip_kick.go +++ b/pkg/transformers/integration_tests/flip_kick.go @@ -65,21 +65,26 @@ var _ = Describe("FlipKick Transformer", func() { Expect(err).NotTo(HaveOccurred()) blockChain, err := getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ + transformer := factories.Transformer{ Config: config, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + }.NewTransformer(db) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flip_kick.FlipKickModel diff --git a/pkg/transformers/integration_tests/flop_kick.go b/pkg/transformers/integration_tests/flop_kick.go index 93690d43..6ecab800 100644 --- a/pkg/transformers/integration_tests/flop_kick.go +++ b/pkg/transformers/integration_tests/flop_kick.go @@ -35,8 +35,13 @@ import ( var _ = Describe("FlopKick Transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + initializer factories.Transformer + fetcher shared.LogFetcher + addresses []common.Address + topics []common.Hash ) BeforeEach(func() { @@ -46,25 +51,33 @@ var _ = Describe("FlopKick Transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = flop_kick.Config + + initializer = factories.Transformer{ + Config: config, + Converter: &flop_kick.FlopKickConverter{}, + Repository: &flop_kick.FlopKickRepository{}, + } + + fetcher = shared.NewFetcher(blockChain) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} }) It("fetches and transforms a FlopKick event from Kovan chain", func() { blockNumber := int64(8672119) - config := flop_kick.Config - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ - Config: config, - Converter: &flop_kick.FlopKickConverter{}, - Repository: &flop_kick.FlopKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flop_kick.Model @@ -82,21 +95,17 @@ var _ = Describe("FlopKick Transformer", func() { It("fetches and transforms another FlopKick event from Kovan chain", func() { blockNumber := int64(8955611) - config := flop_kick.Config - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ - Config: config, - Converter: &flop_kick.FlopKickConverter{}, - Repository: &flop_kick.FlopKickRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []flop_kick.Model diff --git a/pkg/transformers/integration_tests/frob.go b/pkg/transformers/integration_tests/frob.go index 00f31df4..50cde0a0 100644 --- a/pkg/transformers/integration_tests/frob.go +++ b/pkg/transformers/integration_tests/frob.go @@ -33,8 +33,11 @@ import ( var _ = Describe("Frob Transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + fetcher *shared.Fetcher + config shared.TransformerConfig + initializer factories.Transformer ) BeforeEach(func() { @@ -44,25 +47,32 @@ var _ = Describe("Frob Transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + fetcher = shared.NewFetcher(blockChain) + config = frob.FrobConfig + initializer = factories.Transformer{ + Config: config, + Converter: &frob.FrobConverter{}, + Repository: &frob.FrobRepository{}, + } }) It("fetches and transforms a Frob event from Kovan chain", func() { blockNumber := int64(8935258) - config := frob.FrobConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.Transformer{ - Config: config, - Converter: &frob.FrobConverter{}, - Repository: &frob.FrobRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := initializer.NewTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []frob.FrobModel diff --git a/pkg/transformers/integration_tests/helpers.go b/pkg/transformers/integration_tests/helpers.go index 19078a14..f6d00278 100644 --- a/pkg/transformers/integration_tests/helpers.go +++ b/pkg/transformers/integration_tests/helpers.go @@ -42,12 +42,14 @@ func getBlockChain(rpcClient client.RpcClient, ethClient *ethclient.Client) (cor return blockChain, nil } -func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) error { +// Persist the header for a given block to postgres. Returns the header if successful. +func persistHeader(db *postgres.DB, blockNumber int64, blockChain core.BlockChain) (core.Header, error) { header, err := blockChain.GetHeaderByNumber(blockNumber) if err != nil { - return err + return core.Header{}, err } headerRepository := repositories.NewHeaderRepository(db) - _, err = headerRepository.CreateOrUpdateHeader(header) - return err + id, err := headerRepository.CreateOrUpdateHeader(header) + header.Id = id + return header, err } diff --git a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go index c276a3d2..e1b088a3 100644 --- a/pkg/transformers/integration_tests/pit_file_debt_ceiling.go +++ b/pkg/transformers/integration_tests/pit_file_debt_ceiling.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,31 +28,44 @@ import ( ) var _ = Describe("PitFileDebtCeiling LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("fetches and transforms a PitFileDebtCeiling event from Kovan chain", func() { blockNumber := int64(8535578) config := debt_ceiling.DebtCeilingFileConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []debt_ceiling.PitFileDebtCeilingModel diff --git a/pkg/transformers/integration_tests/pit_file_ilk.go b/pkg/transformers/integration_tests/pit_file_ilk.go index cdff6ee7..ab713a94 100644 --- a/pkg/transformers/integration_tests/pit_file_ilk.go +++ b/pkg/transformers/integration_tests/pit_file_ilk.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/pit_file/ilk" @@ -25,31 +28,47 @@ import ( ) var _ = Describe("PitFileIlk LogNoteTransformer", func() { - It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() { - blockNumber := int64(9103223) - config := ilk.IlkFileConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + var ( + db *postgres.DB + blockChain core.BlockChain + initializer factories.LogNoteTransformer + addresses []common.Address + topics []common.Hash + ) + BeforeEach(func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) - - db := test_config.NewTestDB(blockChain.Node()) + db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + config := ilk.IlkFileConfig - err = persistHeader(db, blockNumber, blockChain) - Expect(err).NotTo(HaveOccurred()) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} - initializer := factories.LogNoteTransformer{ + initializer = factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }) + + It("fetches and transforms a Pit.file ilk 'spot' event from Kovan", func() { + blockNumber := int64(9103223) + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber + + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []ilk.PitFileIlkModel @@ -64,29 +83,18 @@ var _ = Describe("PitFileIlk LogNoteTransformer", func() { It("fetches and transforms a Pit.file ilk 'line' event from Kovan", func() { blockNumber := int64(8762253) - config := ilk.IlkFileConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs(addresses, topics, header) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &ilk.PitFileIlkConverter{}, - Repository: &ilk.PitFileIlkRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []ilk.PitFileIlkModel diff --git a/pkg/transformers/integration_tests/price_feeds.go b/pkg/transformers/integration_tests/price_feeds.go index fa8a13e2..fac1f71d 100644 --- a/pkg/transformers/integration_tests/price_feeds.go +++ b/pkg/transformers/integration_tests/price_feeds.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,8 +30,12 @@ import ( var _ = Describe("Price feeds transformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + fetcher *shared.Fetcher + initializer factories.LogNoteTransformer + topics []common.Hash ) BeforeEach(func() { @@ -40,86 +45,94 @@ var _ = Describe("Price feeds transformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = price_feeds.PriceFeedConfig + topics = []common.Hash{common.HexToHash(config.Topic)} + + fetcher = shared.NewFetcher(blockChain) + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &price_feeds.PriceFeedConverter{}, + Repository: &price_feeds.PriceFeedRepository{}, + } }) It("persists a ETH/USD price feed event", func() { blockNumber := int64(8763054) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := price_feeds.PriceFeedConfig - config.ContractAddresses = []string{constants.PipContractAddress} - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + addresses := []string{constants.PipContractAddress} + initializer.Config.ContractAddresses = addresses + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - transformerInitializer := factories.LogNoteTransformer{ - Config: config, - Converter: &price_feeds.PriceFeedConverter{}, - Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(addresses), + topics, + header) + Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel - err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("207.314891143000011198")) - Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) It("persists a MKR/USD price feed event", func() { blockNumber := int64(8763059) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := price_feeds.PriceFeedConfig - config.ContractAddresses = []string{constants.PepContractAddress} - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + addresses := []string{constants.PepContractAddress} + initializer.Config.ContractAddresses = addresses + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - transformerInitializer := factories.LogNoteTransformer{ - Config: config, - Converter: &price_feeds.PriceFeedConverter{}, - Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(addresses), + topics, + header) + Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel - err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("391.803979212000001553")) - Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) It("persists a REP/USD price feed event", func() { blockNumber := int64(8763062) - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - config := price_feeds.PriceFeedConfig - config.ContractAddresses = []string{constants.RepContractAddress} - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + addresses := []string{constants.RepContractAddress} + initializer.Config.ContractAddresses = addresses + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - transformerInitializer := factories.LogNoteTransformer{ - Config: config, - Converter: &price_feeds.PriceFeedConverter{}, - Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - } - transformer := transformerInitializer.NewLogNoteTransformer(db, blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(addresses), + topics, + header) + Expect(err).NotTo(HaveOccurred()) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var model price_feeds.PriceFeedModel - err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, config.StartingBlockNumber) + err = db.Get(&model, `SELECT block_number, medianizer_address, usd_value, tx_idx, raw_log FROM maker.price_feeds WHERE block_number = $1`, initializer.Config.StartingBlockNumber) Expect(err).NotTo(HaveOccurred()) Expect(model.UsdValue).To(Equal("12.816928482699999847")) - Expect(model.MedianizerAddress).To(Equal(config.ContractAddresses[0])) + Expect(model.MedianizerAddress).To(Equal(addresses[0])) }) }) diff --git a/pkg/transformers/integration_tests/tend.go b/pkg/transformers/integration_tests/tend.go index 5fe2084e..ae6938ba 100644 --- a/pkg/transformers/integration_tests/tend.go +++ b/pkg/transformers/integration_tests/tend.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -28,8 +29,13 @@ import ( var _ = Describe("Tend LogNoteTransformer", func() { var ( - db *postgres.DB - blockChain core.BlockChain + db *postgres.DB + blockChain core.BlockChain + config shared.TransformerConfig + fetcher *shared.Fetcher + initializer factories.LogNoteTransformer + addresses []common.Address + topics []common.Hash ) BeforeEach(func() { @@ -39,25 +45,33 @@ var _ = Describe("Tend LogNoteTransformer", func() { Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + + config = tend.TendConfig + + fetcher = shared.NewFetcher(blockChain) + addresses = shared.HexStringsToAddresses(config.ContractAddresses) + topics = []common.Hash{common.HexToHash(config.Topic)} + + initializer = factories.LogNoteTransformer{ + Config: config, + Converter: &tend.TendConverter{}, + Repository: &tend.TendRepository{}, + } }) It("fetches and transforms a Flip Tend event from Kovan chain", func() { blockNumber := int64(8935601) - config := tend.TendConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &tend.TendConverter{}, - Repository: &tend.TendRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel @@ -80,21 +94,17 @@ var _ = Describe("Tend LogNoteTransformer", func() { It("fetches and transforms a subsequent Flip Tend event from Kovan chain for the same auction", func() { blockNumber := int64(8935731) - config := tend.TendConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &tend.TendConverter{}, - Repository: &tend.TendRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel @@ -117,21 +127,17 @@ var _ = Describe("Tend LogNoteTransformer", func() { It("fetches and transforms a Flap Tend event from the Kovan chain", func() { blockNumber := int64(9003177) - config := tend.TendConfig - config.StartingBlockNumber = blockNumber - config.EndingBlockNumber = blockNumber + initializer.Config.StartingBlockNumber = blockNumber + initializer.Config.EndingBlockNumber = blockNumber - err := persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ - Config: config, - Fetcher: &shared.Fetcher{}, - Converter: &tend.TendConverter{}, - Repository: &tend.TendRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + logs, err := fetcher.FetchLogs(addresses, topics, header) + Expect(err).NotTo(HaveOccurred()) + + transformer := initializer.NewLogNoteTransformer(db) + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []tend.TendModel diff --git a/pkg/transformers/integration_tests/vat_flux.go b/pkg/transformers/integration_tests/vat_flux.go index 901e294d..04ecb129 100644 --- a/pkg/transformers/integration_tests/vat_flux.go +++ b/pkg/transformers/integration_tests/vat_flux.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -39,17 +40,24 @@ var _ = Describe("VatFlux LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) initializer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + transformer := initializer.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_flux.VatFluxModel diff --git a/pkg/transformers/integration_tests/vat_fold.go b/pkg/transformers/integration_tests/vat_fold.go index 83096ce4..7458d7dd 100644 --- a/pkg/transformers/integration_tests/vat_fold.go +++ b/pkg/transformers/integration_tests/vat_fold.go @@ -18,39 +18,53 @@ import ( "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/vat_fold" - "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("VatFold Transformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("transforms VatFold log events", func() { blockNumber := int64(9367233) config := vat_fold.VatFoldConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_fold.VatFoldModel diff --git a/pkg/transformers/integration_tests/vat_grab.go b/pkg/transformers/integration_tests/vat_grab.go index f33f0918..f43346f2 100644 --- a/pkg/transformers/integration_tests/vat_grab.go +++ b/pkg/transformers/integration_tests/vat_grab.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" "math/big" . "github.com/onsi/ginkgo" @@ -41,17 +42,23 @@ var _ = Describe("Vat Grab Transformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) transformer := factories.LogNoteTransformer{ Config: config, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer(db, blockChain) + }.NewLogNoteTransformer(db) - err = transformer.Execute() + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_grab.VatGrabModel diff --git a/pkg/transformers/integration_tests/vat_heal.go b/pkg/transformers/integration_tests/vat_heal.go index 61c8ec0d..e787b642 100644 --- a/pkg/transformers/integration_tests/vat_heal.go +++ b/pkg/transformers/integration_tests/vat_heal.go @@ -40,17 +40,23 @@ var _ = Describe("VatHeal Transformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) + Expect(err).NotTo(HaveOccurred()) + + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) transformer := factories.LogNoteTransformer{ Config: config, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer(db, blockChain) + }.NewLogNoteTransformer(db) - err = transformer.Execute() + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_heal.VatHealModel diff --git a/pkg/transformers/integration_tests/vat_init.go b/pkg/transformers/integration_tests/vat_init.go index 010f7429..e31c3aa4 100644 --- a/pkg/transformers/integration_tests/vat_init.go +++ b/pkg/transformers/integration_tests/vat_init.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" @@ -39,17 +40,23 @@ var _ = Describe("VatInit LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_init.VatInitModel diff --git a/pkg/transformers/integration_tests/vat_move.go b/pkg/transformers/integration_tests/vat_move.go index c977e5cc..3ed85452 100644 --- a/pkg/transformers/integration_tests/vat_move.go +++ b/pkg/transformers/integration_tests/vat_move.go @@ -40,17 +40,23 @@ var _ = Describe("VatMove LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResults []vat_move.VatMoveModel diff --git a/pkg/transformers/integration_tests/vat_slip.go b/pkg/transformers/integration_tests/vat_slip.go index 0b82914a..de937cca 100644 --- a/pkg/transformers/integration_tests/vat_slip.go +++ b/pkg/transformers/integration_tests/vat_slip.go @@ -1,6 +1,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -18,31 +19,38 @@ var _ = Describe("Vat slip transformer", func() { blockChain core.BlockChain ) - It("persists vat slip event", func() { + BeforeEach(func() { rpcClient, ethClient, err := getClients(ipc) Expect(err).NotTo(HaveOccurred()) blockChain, err = getBlockChain(rpcClient, ethClient) Expect(err).NotTo(HaveOccurred()) db = test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) + }) + It("persists vat slip event", func() { blockNumber := int64(8953655) config := vat_slip.VatSlipConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_slip.VatSlipConverter{}, Repository: &vat_slip.VatSlipRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) + }.NewLogNoteTransformer(db) - err = transformer.Execute() + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var headerID int64 diff --git a/pkg/transformers/integration_tests/vat_tune.go b/pkg/transformers/integration_tests/vat_tune.go index 59001f80..6a794867 100644 --- a/pkg/transformers/integration_tests/vat_tune.go +++ b/pkg/transformers/integration_tests/vat_tune.go @@ -15,6 +15,7 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" "math/big" . "github.com/onsi/ginkgo" @@ -41,17 +42,23 @@ var _ = Describe("VatTune LogNoteTransformer", func() { db := test_config.NewTestDB(blockChain.Node()) test_config.CleanTestDB(db) - err = persistHeader(db, blockNumber, blockChain) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - initializer := factories.LogNoteTransformer{ + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) + Expect(err).NotTo(HaveOccurred()) + + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vat_tune.VatTuneModel diff --git a/pkg/transformers/integration_tests/vow_flog.go b/pkg/transformers/integration_tests/vow_flog.go index 548086e0..494b4326 100644 --- a/pkg/transformers/integration_tests/vow_flog.go +++ b/pkg/transformers/integration_tests/vow_flog.go @@ -15,8 +15,11 @@ package integration_tests import ( + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" @@ -25,32 +28,43 @@ import ( ) var _ = Describe("VowFlog LogNoteTransformer", func() { + var ( + db *postgres.DB + blockChain core.BlockChain + ) + + BeforeEach(func() { + rpcClient, ethClient, err := getClients(ipc) + Expect(err).NotTo(HaveOccurred()) + blockChain, err = getBlockChain(rpcClient, ethClient) + Expect(err).NotTo(HaveOccurred()) + db = test_config.NewTestDB(blockChain.Node()) + test_config.CleanTestDB(db) + }) + It("transforms VowFlog log events", func() { blockNumber := int64(8946819) config := vow_flog.VowFlogConfig config.StartingBlockNumber = blockNumber config.EndingBlockNumber = blockNumber - rpcClient, ethClient, err := getClients(ipc) - Expect(err).NotTo(HaveOccurred()) - blockChain, err := getBlockChain(rpcClient, ethClient) + header, err := persistHeader(db, blockNumber, blockChain) Expect(err).NotTo(HaveOccurred()) - db := test_config.NewTestDB(blockChain.Node()) - test_config.CleanTestDB(db) - - err = persistHeader(db, blockNumber, blockChain) + fetcher := shared.NewFetcher(blockChain) + logs, err := fetcher.FetchLogs( + shared.HexStringsToAddresses(config.ContractAddresses), + []common.Hash{common.HexToHash(config.Topic)}, + header) Expect(err).NotTo(HaveOccurred()) - Expect(1).To(Equal(1)) - initializer := factories.LogNoteTransformer{ + transformer := factories.LogNoteTransformer{ Config: config, - Fetcher: &shared.Fetcher{}, Converter: &vow_flog.VowFlogConverter{}, Repository: &vow_flog.VowFlogRepository{}, - } - transformer := initializer.NewLogNoteTransformer(db, blockChain) - err = transformer.Execute() + }.NewLogNoteTransformer(db) + + err = transformer.Execute(logs, header) Expect(err).NotTo(HaveOccurred()) var dbResult []vow_flog.VowFlogModel diff --git a/pkg/transformers/pit_file/debt_ceiling/repository.go b/pkg/transformers/pit_file/debt_ceiling/repository.go index 34e398f8..12f69fb2 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository.go @@ -16,7 +16,6 @@ package debt_ceiling import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository PitFileDebtCeilingRepository) MarkHeaderChecked(headerID int64) return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileDebtCeilingChecked) } -func (repository PitFileDebtCeilingRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileDebtCeilingChecked) -} - func (repository *PitFileDebtCeilingRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/pit_file/debt_ceiling/repository_test.go b/pkg/transformers/pit_file/debt_ceiling/repository_test.go index 06a7588b..026d0446 100644 --- a/pkg/transformers/pit_file/debt_ceiling/repository_test.go +++ b/pkg/transformers/pit_file/debt_ceiling/repository_test.go @@ -81,13 +81,4 @@ var _ = Describe("Pit file debt ceiling repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &pitFileDebtCeilingRepository, - RepositoryTwo: &debt_ceiling.PitFileDebtCeilingRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/pit_file/ilk/repository.go b/pkg/transformers/pit_file/ilk/repository.go index c186a638..9a187de7 100644 --- a/pkg/transformers/pit_file/ilk/repository.go +++ b/pkg/transformers/pit_file/ilk/repository.go @@ -16,7 +16,6 @@ package ilk import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -62,10 +61,6 @@ func (repository PitFileIlkRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.PitFileIlkChecked) } -func (repository PitFileIlkRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PitFileIlkChecked) -} - func (repository *PitFileIlkRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/pit_file/ilk/repository_test.go b/pkg/transformers/pit_file/ilk/repository_test.go index 22214f1b..6521ba28 100644 --- a/pkg/transformers/pit_file/ilk/repository_test.go +++ b/pkg/transformers/pit_file/ilk/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Pit file ilk repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &pitFileIlkRepository, - RepositoryTwo: &ilk.PitFileIlkRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/price_feeds/repository.go b/pkg/transformers/price_feeds/repository.go index 59dbab8c..cbb9a012 100644 --- a/pkg/transformers/price_feeds/repository.go +++ b/pkg/transformers/price_feeds/repository.go @@ -16,7 +16,6 @@ package price_feeds import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -58,10 +57,6 @@ func (repository PriceFeedRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.PriceFeedsChecked) } -func (repository PriceFeedRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.PriceFeedsChecked) -} - func (repository *PriceFeedRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/price_feeds/repository_test.go b/pkg/transformers/price_feeds/repository_test.go index 52765fd4..f58aef0d 100644 --- a/pkg/transformers/price_feeds/repository_test.go +++ b/pkg/transformers/price_feeds/repository_test.go @@ -82,13 +82,4 @@ var _ = Describe("Price feeds repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &priceFeedRepository, - RepositoryTwo: &price_feeds.PriceFeedRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/shared/log_chunker.go b/pkg/transformers/shared/log_chunker.go new file mode 100644 index 00000000..b7373ff8 --- /dev/null +++ b/pkg/transformers/shared/log_chunker.go @@ -0,0 +1,65 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +type Chunker interface { + AddConfigs(transformerConfigs []TransformerConfig) + ChunkLogs(logs []types.Log) map[string][]types.Log +} + +type LogChunker struct { + AddressToNames map[string][]string + NameToTopic0 map[string]common.Hash +} + +// Returns a new log chunker with initialised maps. +// Needs to have configs added with `AddConfigs` to consider logs for the respective transformer. +func NewLogChunker() *LogChunker { + return &LogChunker{ + AddressToNames: map[string][]string{}, + NameToTopic0: map[string]common.Hash{}, + } +} + +// Configures the chunker by adding more addreses and topics to consider. +func (chunker *LogChunker) AddConfigs(transformerConfigs []TransformerConfig) { + for _, config := range transformerConfigs { + for _, address := range config.ContractAddresses { + chunker.AddressToNames[address] = append(chunker.AddressToNames[address], config.TransformerName) + chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic) + } + } +} + +// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers +func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log { + chunks := map[string][]types.Log{} + for _, log := range logs { + // Topic0 is not unique to each transformer, also need to consider the contract address + relevantTransformers := chunker.AddressToNames[log.Address.String()] + + for _, transformer := range relevantTransformers { + if chunker.NameToTopic0[transformer] == log.Topics[0] { + chunks[transformer] = append(chunks[transformer], log) + } + } + } + return chunks +} diff --git a/pkg/transformers/shared/log_chunker_test.go b/pkg/transformers/shared/log_chunker_test.go new file mode 100644 index 00000000..7dfd3d69 --- /dev/null +++ b/pkg/transformers/shared/log_chunker_test.go @@ -0,0 +1,137 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +var _ = Describe("Log chunker", func() { + var ( + configs []shared.TransformerConfig + chunker *shared.LogChunker + ) + + BeforeEach(func() { + configA := shared.TransformerConfig{ + TransformerName: "TransformerA", + ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"}, + Topic: "0xA", + } + configB := shared.TransformerConfig{ + TransformerName: "TransformerB", + ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"}, + Topic: "0xB", + } + + configC := shared.TransformerConfig{ + TransformerName: "TransformerC", + ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"}, + Topic: "0xC", + } + + configs = []shared.TransformerConfig{configA, configB, configC} + chunker = shared.NewLogChunker() + chunker.AddConfigs(configs) + }) + + Describe("initialisation", func() { + It("creates lookup maps correctly", func() { + Expect(chunker.AddressToNames).To(Equal(map[string][]string{ + "0x00000000000000000000000000000000000000A1": []string{"TransformerA"}, + "0x00000000000000000000000000000000000000A2": []string{"TransformerA", "TransformerC"}, + "0x00000000000000000000000000000000000000B1": []string{"TransformerB"}, + })) + + Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{ + "TransformerA": common.HexToHash("0xA"), + "TransformerB": common.HexToHash("0xB"), + "TransformerC": common.HexToHash("0xC"), + })) + }) + }) + + Describe("AddConfigs", func() { + It("can add more configs later", func() { + configD := shared.TransformerConfig{ + TransformerName: "TransformerD", + ContractAddresses: []string{"0x000000000000000000000000000000000000000D"}, + Topic: "0xD", + } + chunker.AddConfigs([]shared.TransformerConfig{configD}) + + Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"})) + Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD"))) + }) + }) + + Describe("ChunkLogs", func() { + It("only associates logs with relevant topic0 and address to transformers", func() { + logs := []types.Log{log1, log2, log3, log4, log5} + chunks := chunker.ChunkLogs(logs) + + Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4))) + Expect(chunks["TransformerB"]).To(BeEmpty()) + Expect(chunks["TransformerC"]).To(ContainElement(log5)) + }) + }) +}) + +var ( + // Match TransformerA + log1 = types.Log{ + Address: common.HexToAddress("0xA1"), + Topics: []common.Hash{ + common.HexToHash("0xA"), + common.HexToHash("0xLogTopic1"), + }, + } + // Match TransformerA address, but not topic0 + log2 = types.Log{ + Address: common.HexToAddress("0xA1"), + Topics: []common.Hash{ + common.HexToHash("0xB"), + common.HexToHash("0xLogTopic2"), + }, + } + // Match TransformerA topic, but TransformerB address + log3 = types.Log{ + Address: common.HexToAddress("0xB1"), + Topics: []common.Hash{ + common.HexToHash("0xA"), + common.HexToHash("0xLogTopic3"), + }, + } + // Match TransformerA, with the other address + log4 = types.Log{ + Address: common.HexToAddress("0xA2"), + Topics: []common.Hash{ + common.HexToHash("0xA"), + common.HexToHash("0xLogTopic4"), + }, + } + // Match TransformerC, which shares address with TransformerA + log5 = types.Log{ + Address: common.HexToAddress("0xA2"), + Topics: []common.Hash{ + common.HexToHash("0xC"), + common.HexToHash("0xLogTopic5"), + }, + } +) diff --git a/pkg/transformers/shared/log_fetcher.go b/pkg/transformers/shared/log_fetcher.go index 0e6ad7ce..11081a24 100644 --- a/pkg/transformers/shared/log_fetcher.go +++ b/pkg/transformers/shared/log_fetcher.go @@ -23,45 +23,34 @@ import ( ) type LogFetcher interface { - FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) -} - -type SettableLogFetcher interface { - LogFetcher - SetBC(bc core.BlockChain) + FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) } type Fetcher struct { blockChain core.BlockChain } -func (fetcher *Fetcher) SetBC(bc core.BlockChain) { - fetcher.blockChain = bc -} - -func NewFetcher(blockchain core.BlockChain) Fetcher { - return Fetcher{ +func NewFetcher(blockchain core.BlockChain) *Fetcher { + return &Fetcher{ blockChain: blockchain, } } -func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { - addresses := hexStringsToAddresses(contractAddresses) +// Checks all topic0s, on all addresses, fetching matching logs for the given header +func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) { blockHash := common.HexToHash(header.Hash) query := ethereum.FilterQuery{ BlockHash: &blockHash, Addresses: addresses, - Topics: topics, - } - return fetcher.blockChain.GetEthLogsWithCustomQuery(query) -} - -func hexStringsToAddresses(hexStrings []string) []common.Address { - var addresses []common.Address - for _, hexString := range hexStrings { - address := common.HexToAddress(hexString) - addresses = append(addresses, address) + // Search for _any_ of the topics in topic0 position; see docs on `FilterQuery` + Topics: [][]common.Hash{topic0s}, } - return addresses + logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query) + if err != nil { + // TODO review aggregate fetching error handling + return []types.Log{}, err + } + + return logs, nil } diff --git a/pkg/transformers/shared/log_fetcher_test.go b/pkg/transformers/shared/log_fetcher_test.go index 8e5aa7fa..4a0c55fa 100644 --- a/pkg/transformers/shared/log_fetcher_test.go +++ b/pkg/transformers/shared/log_fetcher_test.go @@ -32,8 +32,12 @@ var _ = Describe("Fetcher", func() { fetcher := shared.NewFetcher(blockChain) header := fakes.FakeHeader - addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"} - topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}} + addresses := []common.Address{ + common.HexToAddress("0xfakeAddress"), + common.HexToAddress("0xanotherFakeAddress"), + } + + topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})} _, err := fetcher.FetchLogs(addresses, topicZeros, header) @@ -45,7 +49,7 @@ var _ = Describe("Fetcher", func() { expectedQuery := ethereum.FilterQuery{ BlockHash: &blockHash, Addresses: []common.Address{address1, address2}, - Topics: topicZeros, + Topics: [][]common.Hash{topicZeros}, } blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) }) @@ -55,7 +59,7 @@ var _ = Describe("Fetcher", func() { blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) fetcher := shared.NewFetcher(blockChain) - _, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{}) + _, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{}) Expect(err).To(HaveOccurred()) Expect(err).To(MatchError(fakes.FakeError)) diff --git a/pkg/transformers/shared/repository.go b/pkg/transformers/shared/repository.go index d0c6b5f6..65c01c99 100644 --- a/pkg/transformers/shared/repository.go +++ b/pkg/transformers/shared/repository.go @@ -1,7 +1,10 @@ package shared import ( + "bytes" "database/sql" + "database/sql/driver" + "fmt" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -22,7 +25,8 @@ func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersCo return err } -func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) { +// Treats a header as missing if it's not in the headers table, or not checked for some log type +func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { var result []core.Header var query string var err error @@ -30,14 +34,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D if endingBlockNumber == -1 { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) + WHERE (header_id ISNULL OR ` + notCheckedSQL + `) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2` err = db.Select(&result, query, startingBlockNumber, db.Node.ID) } else { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers LEFT JOIN checked_headers on headers.id = header_id - WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) + WHERE (header_id ISNULL OR ` + notCheckedSQL + `) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3` @@ -46,3 +50,53 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D return result, err } + +func GetCheckedColumnNames(db *postgres.DB) ([]string, error) { + // Query returns `[]driver.Value`, nullable polymorphic interface + var queryResult []driver.Value + columnNamesQuery := + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'checked_headers' + AND column_name ~ '_checked';` + + err := db.Select(&queryResult, columnNamesQuery) + if err != nil { + return []string{}, err + } + + // Transform column names from `driver.Value` to strings + var columnNames []string + for _, result := range queryResult { + if columnName, ok := result.(string); ok { + columnNames = append(columnNames, columnName) + } else { + return []string{}, fmt.Errorf("incorrect value for checked_headers column name") + } + } + + return columnNames, nil +} + +// Builds a SQL string that checks if any column value is FALSE, given the column names. +// Defaults to FALSE when no columns are provided. +// Ex: ["columnA", "columnB"] => "NOT (columnA AND columnB)" +// [] => "FALSE" +func CreateNotCheckedSQL(boolColumns []string) string { + var result bytes.Buffer + + if len(boolColumns) == 0 { + return "FALSE" + } + + result.WriteString("NOT (") + // Loop excluding last column name + for _, column := range boolColumns[:len(boolColumns)-1] { + result.WriteString(fmt.Sprintf("%v AND ", column)) + } + + // No trailing "OR" for the last column name + result.WriteString(fmt.Sprintf("%v)", boolColumns[len(boolColumns)-1])) + + return result.String() +} diff --git a/pkg/transformers/shared/repository_utility_test.go b/pkg/transformers/shared/repository_utility_test.go new file mode 100644 index 00000000..2699108e --- /dev/null +++ b/pkg/transformers/shared/repository_utility_test.go @@ -0,0 +1,170 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" + "github.com/vulcanize/vulcanizedb/test_config" + "math/rand" +) + +var _ = Describe("Repository utilities", func() { + Describe("MissingHeaders", func() { + var ( + db *postgres.DB + headerRepository datastore.HeaderRepository + startingBlockNumber int64 + endingBlockNumber int64 + eventSpecificBlockNumber int64 + blockNumbers []int64 + headerIDs []int64 + notCheckedSQL string + err error + ) + + BeforeEach(func() { + db = test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + headerRepository = repositories.NewHeaderRepository(db) + + columnNames, err := shared.GetCheckedColumnNames(db) + Expect(err).NotTo(HaveOccurred()) + notCheckedSQL = shared.CreateNotCheckedSQL(columnNames) + + startingBlockNumber = rand.Int63() + eventSpecificBlockNumber = startingBlockNumber + 1 + endingBlockNumber = startingBlockNumber + 2 + outOfRangeBlockNumber := endingBlockNumber + 1 + + blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber} + + headerIDs = []int64{} + for _, n := range blockNumbers { + headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) + headerIDs = append(headerIDs, headerID) + Expect(err).NotTo(HaveOccurred()) + } + }) + + It("only treats headers as checked if the event specific logs have been checked", func() { + _, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) + Expect(err).NotTo(HaveOccurred()) + + headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(headers)).To(Equal(3)) + Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) + Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) + Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) + }) + + It("only returns headers associated with the current node", func() { + dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) + headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) + for _, n := range blockNumbers { + _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10)) + Expect(err).NotTo(HaveOccurred()) + } + + Expect(err).NotTo(HaveOccurred()) + nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeOneMissingHeaders)).To(Equal(3)) + Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) + Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) + Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) + + nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) + Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10))) + Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10))) + }) + }) + + Describe("GetCheckedColumnNames", func() { + It("gets the column names from checked_headers", func() { + db := test_config.NewTestDB(test_config.NewTestNode()) + test_config.CleanTestDB(db) + expectedColumnNames := getExpectedColumnNames() + actualColumnNames, err := shared.GetCheckedColumnNames(db) + Expect(err).NotTo(HaveOccurred()) + Expect(actualColumnNames).To(Equal(expectedColumnNames)) + }) + }) + + Describe("CreateNotCheckedSQL", func() { + It("generates a correct SQL string for one column", func() { + columns := []string{"columnA"} + expected := "NOT (columnA)" + actual := shared.CreateNotCheckedSQL(columns) + Expect(actual).To(Equal(expected)) + }) + + It("generates a correct SQL string for several columns", func() { + columns := []string{"columnA", "columnB"} + expected := "NOT (columnA AND columnB)" + actual := shared.CreateNotCheckedSQL(columns) + Expect(actual).To(Equal(expected)) + }) + + It("defaults to FALSE when there are no columns", func() { + expected := "FALSE" + actual := shared.CreateNotCheckedSQL([]string{}) + Expect(actual).To(Equal(expected)) + }) + }) +}) + +func getExpectedColumnNames() []string { + return []string{ + "price_feeds_checked", + "flip_kick_checked", + "frob_checked", + "tend_checked", + "bite_checked", + "dent_checked", + "pit_file_debt_ceiling_checked", + "pit_file_ilk_checked", + "vat_init_checked", + "drip_file_ilk_checked", + "drip_file_repo_checked", + "drip_file_vow_checked", + "deal_checked", + "drip_drip_checked", + "cat_file_chop_lump_checked", + "cat_file_flip_checked", + "cat_file_pit_vow_checked", + "flop_kick_checked", + "vat_move_checked", + "vat_fold_checked", + "vat_heal_checked", + "vat_toll_checked", + "vat_tune_checked", + "vat_grab_checked", + "vat_flux_checked", + "vat_slip_checked", + "vow_flog_checked", + "flap_kick_checked", + } +} diff --git a/pkg/transformers/shared/transformer.go b/pkg/transformers/shared/transformer.go index 267077ea..0cc510d4 100644 --- a/pkg/transformers/shared/transformer.go +++ b/pkg/transformers/shared/transformer.go @@ -16,16 +16,18 @@ package shared import ( "github.com/ethereum/go-ethereum/common" - + "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) type Transformer interface { - Execute() error + Execute(logs []types.Log, header core.Header) error + GetConfig() TransformerConfig } -type TransformerInitializer func(db *postgres.DB, blockChain core.BlockChain) Transformer +type TransformerInitializer func(db *postgres.DB) Transformer type TransformerConfig struct { TransformerName string @@ -45,3 +47,10 @@ func HexToString(byteString string) string { value := common.HexToHash(byteString) return value.Big().String() } + +func HexStringsToAddresses(strings []string) (addresses []common.Address) { + for _, hexString := range strings { + addresses = append(addresses, common.HexToAddress(hexString)) + } + return +} diff --git a/pkg/transformers/tend/repository.go b/pkg/transformers/tend/repository.go index 61982e12..17620a00 100644 --- a/pkg/transformers/tend/repository.go +++ b/pkg/transformers/tend/repository.go @@ -16,7 +16,6 @@ package tend import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -68,10 +67,6 @@ func (repository TendRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.TendChecked) } -func (repository TendRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.TendChecked) -} - func (repository *TendRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/tend/repository_test.go b/pkg/transformers/tend/repository_test.go index dae2d4ba..0753524f 100644 --- a/pkg/transformers/tend/repository_test.go +++ b/pkg/transformers/tend/repository_test.go @@ -94,13 +94,4 @@ var _ = Describe("TendRepository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &tendRepository, - RepositoryTwo: &tend.TendRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/test_data/mocks/log_fetcher.go b/pkg/transformers/test_data/mocks/log_fetcher.go index 47ce03aa..ff3db503 100644 --- a/pkg/transformers/test_data/mocks/log_fetcher.go +++ b/pkg/transformers/test_data/mocks/log_fetcher.go @@ -21,18 +21,20 @@ import ( ) type MockLogFetcher struct { - FetchedContractAddresses [][]string + FetchedContractAddresses [][]common.Address FetchedTopics [][]common.Hash FetchedBlocks []int64 fetcherError error FetchedLogs []types.Log SetBcCalled bool + FetchLogsCalled bool } -func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { +func (mlf *MockLogFetcher) FetchLogs(contractAddresses []common.Address, topics []common.Hash, header core.Header) ([]types.Log, error) { mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses) - mlf.FetchedTopics = topics + mlf.FetchedTopics = [][]common.Hash{topics} mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber) + mlf.FetchLogsCalled = true return mlf.FetchedLogs, mlf.fetcherError } diff --git a/pkg/transformers/test_data/mocks/mock_watcher_repository.go b/pkg/transformers/test_data/mocks/mock_watcher_repository.go new file mode 100644 index 00000000..323fdddf --- /dev/null +++ b/pkg/transformers/test_data/mocks/mock_watcher_repository.go @@ -0,0 +1,53 @@ +package mocks + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type MockWatcherRepository struct { + ReturnCheckedColumnNames []string + GetCheckedColumnNamesError error + GetCheckedColumnNamesCalled bool + + ReturnNotCheckedSQL string + CreateNotCheckedSQLCalled bool + + ReturnMissingHeaders []core.Header + MissingHeadersError error + MissingHeadersCalled bool +} + +func (repository *MockWatcherRepository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) { + repository.GetCheckedColumnNamesCalled = true + if repository.GetCheckedColumnNamesError != nil { + return []string{}, repository.GetCheckedColumnNamesError + } + + return repository.ReturnCheckedColumnNames, nil +} + +func (repository *MockWatcherRepository) SetCheckedColumnNames(checkedColumnNames []string) { + repository.ReturnCheckedColumnNames = checkedColumnNames +} + +func (repository *MockWatcherRepository) CreateNotCheckedSQL(boolColumns []string) string { + repository.CreateNotCheckedSQLCalled = true + return repository.ReturnNotCheckedSQL +} + +func (repository *MockWatcherRepository) SetNotCheckedSQL(notCheckedSql string) { + repository.ReturnNotCheckedSQL = notCheckedSql +} + +func (repository *MockWatcherRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) { + if repository.MissingHeadersError != nil { + return []core.Header{}, repository.MissingHeadersError + } + repository.MissingHeadersCalled = true + return repository.ReturnMissingHeaders, nil +} + +func (repository *MockWatcherRepository) SetMissingHeaders(headers []core.Header) { + repository.ReturnMissingHeaders = headers +} diff --git a/pkg/transformers/test_data/mocks/repository.go b/pkg/transformers/test_data/mocks/repository.go index 1cf5bf0a..0a998401 100644 --- a/pkg/transformers/test_data/mocks/repository.go +++ b/pkg/transformers/test_data/mocks/repository.go @@ -3,7 +3,6 @@ package mocks import ( . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) @@ -12,8 +11,6 @@ type MockRepository struct { markHeaderCheckedError error MarkHeaderCheckedPassedHeaderIDs []int64 CreatedHeaderIds []int64 - missingHeaders []core.Header - missingHeadersError error PassedStartingBlockNumber int64 PassedEndingBlockNumber int64 PassedHeaderID int64 @@ -36,24 +33,10 @@ func (repository *MockRepository) MarkHeaderChecked(headerID int64) error { return repository.markHeaderCheckedError } -func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - repository.PassedStartingBlockNumber = startingBlockNumber - repository.PassedEndingBlockNumber = endingBlockNumber - return repository.missingHeaders, repository.missingHeadersError -} - func (repository *MockRepository) SetDB(db *postgres.DB) { repository.SetDbCalled = true } -func (repository *MockRepository) SetMissingHeadersError(e error) { - repository.missingHeadersError = e -} - -func (repository *MockRepository) SetMissingHeaders(headers []core.Header) { - repository.missingHeaders = headers -} - func (repository *MockRepository) SetMarkHeaderCheckedError(e error) { repository.markHeaderCheckedError = e } diff --git a/pkg/transformers/test_data/mocks/transformer.go b/pkg/transformers/test_data/mocks/transformer.go new file mode 100644 index 00000000..0b054567 --- /dev/null +++ b/pkg/transformers/test_data/mocks/transformer.go @@ -0,0 +1,44 @@ +package mocks + +import ( + "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" +) + +type MockTransformer struct { + ExecuteWasCalled bool + ExecuteError error + PassedLogs []types.Log + PassedHeader core.Header + config shared.TransformerConfig +} + +func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error { + if mh.ExecuteError != nil { + return mh.ExecuteError + } + mh.ExecuteWasCalled = true + mh.PassedLogs = logs + mh.PassedHeader = header + return nil +} + +func (mh *MockTransformer) GetConfig() shared.TransformerConfig { + return mh.config +} + +func (mh *MockTransformer) SetTransformerConfig(config shared.TransformerConfig) { + mh.config = config +} + +func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared.Transformer { + return mh +} + +var FakeTransformerConfig = shared.TransformerConfig{ + TransformerName: "FakeTransformer", + ContractAddresses: []string{"FakeAddress"}, + Topic: "FakeTopic", +} diff --git a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go index 65ba206c..745dcd3d 100644 --- a/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go +++ b/pkg/transformers/test_data/shared_behaviors/repository_behaviors.go @@ -17,7 +17,6 @@ package shared_behaviors import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -25,7 +24,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/test_config" - "math/rand" ) var ( @@ -61,7 +59,7 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { var logEventModel = inputs.TestModel BeforeEach(func() { - headerRepository = GetHeaderRepository() + headerRepository = getHeaderRepository() headerID, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) }) @@ -132,94 +130,13 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) { }) } -func SharedRepositoryMissingHeadersBehaviors(inputs *MissingHeadersBehaviorInputs) { - Describe("MissingHeaders", func() { - var ( - repository = inputs.Repository - startingBlockNumber int64 - endingBlockNumber int64 - eventSpecificBlockNumber int64 - blockNumbers []int64 - headerIDs []int64 - ) - - BeforeEach(func() { - headerRepository = GetHeaderRepository() - startingBlockNumber = rand.Int63() - eventSpecificBlockNumber = startingBlockNumber + 1 - endingBlockNumber = startingBlockNumber + 2 - outOfRangeBlockNumber := endingBlockNumber + 1 - - blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber} - - headerIDs = []int64{} - for _, n := range blockNumbers { - headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) - headerIDs = append(headerIDs, headerID) - Expect(err).NotTo(HaveOccurred()) - } - }) - - It("returns headers that haven't been checked", func() { - err := repository.MarkHeaderChecked(headerIDs[1]) - Expect(err).NotTo(HaveOccurred()) - - headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) - - Expect(err).NotTo(HaveOccurred()) - Expect(len(headers)).To(Equal(2)) - Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) - Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber))) - }) - - It("only treats headers as checked if the event specific logs have been checked", func() { - _, err := db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1]) - Expect(err).NotTo(HaveOccurred()) - - headers, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) - - Expect(err).NotTo(HaveOccurred()) - Expect(len(headers)).To(Equal(3)) - Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) - Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) - Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber))) - }) - - It("only returns headers associated with the current node", func() { - dbTwo := test_config.NewTestDB(core.Node{ID: "second"}) - headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo) - for _, n := range blockNumbers { - _, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n)) - Expect(err).NotTo(HaveOccurred()) - } - repositoryTwo := inputs.RepositoryTwo - repositoryTwo.SetDB(dbTwo) - - err := repository.MarkHeaderChecked(headerIDs[0]) - Expect(err).NotTo(HaveOccurred()) - nodeOneMissingHeaders, err := repository.MissingHeaders(startingBlockNumber, endingBlockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(nodeOneMissingHeaders)).To(Equal(2)) - Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - - nodeTwoMissingHeaders, err := repositoryTwo.MissingHeaders(startingBlockNumber, endingBlockNumber) - Expect(err).NotTo(HaveOccurred()) - Expect(len(nodeTwoMissingHeaders)).To(Equal(3)) - Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - Expect(nodeTwoMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber))) - }) - }) -} - func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehaviorInputs) { var repository = inputs.Repository var checkedHeaderColumn = inputs.CheckedHeaderColumnName Describe("MarkHeaderChecked", func() { BeforeEach(func() { - headerRepository = GetHeaderRepository() + headerRepository = getHeaderRepository() headerId, err = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader) Expect(err).NotTo(HaveOccurred()) }) @@ -257,7 +174,7 @@ func SharedRepositoryMarkHeaderCheckedBehaviors(inputs *MarkedHeaderCheckedBehav }) } -func GetHeaderRepository() repositories.HeaderRepository { +func getHeaderRepository() repositories.HeaderRepository { db = test_config.NewTestDB(test_config.NewTestNode()) test_config.CleanTestDB(db) diff --git a/pkg/transformers/transformers.go b/pkg/transformers/transformers.go index f79ac036..6b0b7938 100644 --- a/pkg/transformers/transformers.go +++ b/pkg/transformers/transformers.go @@ -47,233 +47,223 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/transformers/vow_flog" ) +// Custom event transformers var ( - BiteTransformerInitializer = factories.Transformer{ + BiteTransformer = factories.Transformer{ Config: bite.BiteConfig, Converter: &bite.BiteConverter{}, Repository: &bite.BiteRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewTransformer + } - CatFileChopLumpTransformerInitializer = factories.LogNoteTransformer{ - Config: chop_lump.CatFileChopLumpConfig, - Converter: &chop_lump.CatFileChopLumpConverter{}, - Repository: &chop_lump.CatFileChopLumpRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - CatFileFlipTransformerInitializer = factories.LogNoteTransformer{ - Config: flip.CatFileFlipConfig, - Converter: &flip.CatFileFlipConverter{}, - Repository: &flip.CatFileFlipRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - CatFilePitVowTransformerInitializer = factories.LogNoteTransformer{ - Config: pit_vow.CatFilePitVowConfig, - Converter: &pit_vow.CatFilePitVowConverter{}, - Repository: &pit_vow.CatFilePitVowRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - DealTransformerInitializer = factories.LogNoteTransformer{ - Config: deal.DealConfig, - Converter: &deal.DealConverter{}, - Repository: &deal.DealRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - DentTransformerInitializer = factories.LogNoteTransformer{ - Config: dent.DentConfig, - Converter: &dent.DentConverter{}, - Repository: &dent.DentRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - DripDripTransformerInitializer = factories.LogNoteTransformer{ - Config: drip_drip.DripDripConfig, - Converter: &drip_drip.DripDripConverter{}, - Repository: &drip_drip.DripDripRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - DripFileIlkTransformerInitializer = factories.LogNoteTransformer{ - Config: ilk2.DripFileIlkConfig, - Converter: &ilk2.DripFileIlkConverter{}, - Repository: &ilk2.DripFileIlkRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - DripFileRepoTransformerInitializer = factories.LogNoteTransformer{ - Config: repo.DripFileRepoConfig, - Converter: &repo.DripFileRepoConverter{}, - Repository: &repo.DripFileRepoRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - DripFileVowTransfromerInitializer = factories.LogNoteTransformer{ - Config: vow.DripFileVowConfig, - Converter: &vow.DripFileVowConverter{}, - Repository: &vow.DripFileVowRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - FlapKickTransformerInitializer = factories.Transformer{ + FlapKickTransformer = factories.Transformer{ Config: flap_kick.FlapKickConfig, Converter: &flap_kick.FlapKickConverter{}, Repository: &flap_kick.FlapKickRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewTransformer + } - FlipKickTransformerInitializer = factories.Transformer{ + FlipKickTransformer = factories.Transformer{ Config: flip_kick.FlipKickConfig, Converter: &flip_kick.FlipKickConverter{}, Repository: &flip_kick.FlipKickRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewTransformer + } - FlogTransformerInitializer = factories.LogNoteTransformer{ - Config: vow_flog.VowFlogConfig, - Converter: &vow_flog.VowFlogConverter{}, - Repository: &vow_flog.VowFlogRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer - - FrobTransformerInitializer = factories.Transformer{ + FrobTransformer = factories.Transformer{ Config: frob.FrobConfig, Converter: &frob.FrobConverter{}, Repository: &frob.FrobRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewTransformer + } - FlopKickTransformerInitializer = factories.Transformer{ + FlopKickTransformer = factories.Transformer{ Config: flop_kick.Config, Converter: &flop_kick.FlopKickConverter{}, Repository: &flop_kick.FlopKickRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewTransformer + } - PitFileDebtCeilingTransformerInitializer = factories.LogNoteTransformer{ + customEventTransformers = []factories.Transformer{ + BiteTransformer, + FlapKickTransformer, + FlipKickTransformer, + FrobTransformer, + FlopKickTransformer, + } +) + +// LogNote transformers +var ( + CatFileChopLumpTransformer = factories.LogNoteTransformer{ + Config: chop_lump.CatFileChopLumpConfig, + Converter: &chop_lump.CatFileChopLumpConverter{}, + Repository: &chop_lump.CatFileChopLumpRepository{}, + } + + CatFileFlipTransformer = factories.LogNoteTransformer{ + Config: flip.CatFileFlipConfig, + Converter: &flip.CatFileFlipConverter{}, + Repository: &flip.CatFileFlipRepository{}, + } + + CatFilePitVowTransformer = factories.LogNoteTransformer{ + Config: pit_vow.CatFilePitVowConfig, + Converter: &pit_vow.CatFilePitVowConverter{}, + Repository: &pit_vow.CatFilePitVowRepository{}, + } + + DealTransformer = factories.LogNoteTransformer{ + Config: deal.DealConfig, + Converter: &deal.DealConverter{}, + Repository: &deal.DealRepository{}, + } + + DentTransformer = factories.LogNoteTransformer{ + Config: dent.DentConfig, + Converter: &dent.DentConverter{}, + Repository: &dent.DentRepository{}, + } + + DripDripTransformer = factories.LogNoteTransformer{ + Config: drip_drip.DripDripConfig, + Converter: &drip_drip.DripDripConverter{}, + Repository: &drip_drip.DripDripRepository{}, + } + + DripFileIlkTransformer = factories.LogNoteTransformer{ + Config: ilk2.DripFileIlkConfig, + Converter: &ilk2.DripFileIlkConverter{}, + Repository: &ilk2.DripFileIlkRepository{}, + } + + DripFileRepoTransformer = factories.LogNoteTransformer{ + Config: repo.DripFileRepoConfig, + Converter: &repo.DripFileRepoConverter{}, + Repository: &repo.DripFileRepoRepository{}, + } + + DripFileVowTransfromer = factories.LogNoteTransformer{ + Config: vow.DripFileVowConfig, + Converter: &vow.DripFileVowConverter{}, + Repository: &vow.DripFileVowRepository{}, + } + + FlogTransformer = factories.LogNoteTransformer{ + Config: vow_flog.VowFlogConfig, + Converter: &vow_flog.VowFlogConverter{}, + Repository: &vow_flog.VowFlogRepository{}, + } + + PitFileDebtCeilingTransformer = factories.LogNoteTransformer{ Config: debt_ceiling.DebtCeilingFileConfig, Converter: &debt_ceiling.PitFileDebtCeilingConverter{}, Repository: &debt_ceiling.PitFileDebtCeilingRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - PitFileIlkTransformerInitializer = factories.LogNoteTransformer{ + PitFileIlkTransformer = factories.LogNoteTransformer{ Config: ilk.IlkFileConfig, Converter: &ilk.PitFileIlkConverter{}, Repository: &ilk.PitFileIlkRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - PriceFeedTransformerInitializer = factories.LogNoteTransformer{ + PriceFeedTransformer = factories.LogNoteTransformer{ Config: price_feeds.PriceFeedConfig, Converter: &price_feeds.PriceFeedConverter{}, Repository: &price_feeds.PriceFeedRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - TendTransformerInitializer = factories.LogNoteTransformer{ + TendTransformer = factories.LogNoteTransformer{ Config: tend.TendConfig, Converter: &tend.TendConverter{}, Repository: &tend.TendRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatInitTransformerInitializer = factories.LogNoteTransformer{ + VatInitTransformer = factories.LogNoteTransformer{ Config: vat_init.VatInitConfig, Converter: &vat_init.VatInitConverter{}, Repository: &vat_init.VatInitRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatGrabTransformerInitializer = factories.LogNoteTransformer{ + VatGrabTransformer = factories.LogNoteTransformer{ Config: vat_grab.VatGrabConfig, Converter: &vat_grab.VatGrabConverter{}, Repository: &vat_grab.VatGrabRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatFoldTransformerInitializer = factories.LogNoteTransformer{ + VatFoldTransformer = factories.LogNoteTransformer{ Config: vat_fold.VatFoldConfig, Converter: &vat_fold.VatFoldConverter{}, Repository: &vat_fold.VatFoldRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatHealTransformerInitializer = factories.LogNoteTransformer{ + VatHealTransformer = factories.LogNoteTransformer{ Config: vat_heal.VatHealConfig, Converter: &vat_heal.VatHealConverter{}, Repository: &vat_heal.VatHealRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatMoveTransformerInitializer = factories.LogNoteTransformer{ + VatMoveTransformer = factories.LogNoteTransformer{ Config: vat_move.VatMoveConfig, Converter: &vat_move.VatMoveConverter{}, Repository: &vat_move.VatMoveRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatSlipTransformerInitializer = factories.LogNoteTransformer{ + VatSlipTransformer = factories.LogNoteTransformer{ Config: vat_slip.VatSlipConfig, Converter: &vat_slip.VatSlipConverter{}, Repository: &vat_slip.VatSlipRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatTollTransformerInitializer = factories.LogNoteTransformer{ + VatTollTransformer = factories.LogNoteTransformer{ Config: vat_toll.VatTollConfig, Converter: &vat_toll.VatTollConverter{}, Repository: &vat_toll.VatTollRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatTuneTransformerInitializer = factories.LogNoteTransformer{ + VatTuneTransformer = factories.LogNoteTransformer{ Config: vat_tune.VatTuneConfig, Converter: &vat_tune.VatTuneConverter{}, Repository: &vat_tune.VatTuneRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } - VatFluxTransformerInitializer = factories.LogNoteTransformer{ + VatFluxTransformer = factories.LogNoteTransformer{ Config: vat_flux.VatFluxConfig, Converter: &vat_flux.VatFluxConverter{}, Repository: &vat_flux.VatFluxRepository{}, - Fetcher: &shared.Fetcher{}, - }.NewLogNoteTransformer + } + + logNoteTransformers = []factories.LogNoteTransformer{ + CatFileChopLumpTransformer, + CatFileFlipTransformer, + CatFilePitVowTransformer, + DealTransformer, + DentTransformer, + DripDripTransformer, + DripFileIlkTransformer, + DripFileRepoTransformer, + DripFileVowTransfromer, + FlogTransformer, + PitFileDebtCeilingTransformer, + PitFileIlkTransformer, + PriceFeedTransformer, + TendTransformer, + VatInitTransformer, + VatGrabTransformer, + VatFoldTransformer, + VatHealTransformer, + VatMoveTransformer, + VatSlipTransformer, + VatTollTransformer, + VatTuneTransformer, + VatFluxTransformer, + } ) -func TransformerInitializers() []shared.TransformerInitializer { - return []shared.TransformerInitializer{ - BiteTransformerInitializer, - CatFileChopLumpTransformerInitializer, - CatFileFlipTransformerInitializer, - CatFilePitVowTransformerInitializer, - DealTransformerInitializer, - DentTransformerInitializer, - DripDripTransformerInitializer, - DripFileIlkTransformerInitializer, - DripFileVowTransfromerInitializer, - DripFileRepoTransformerInitializer, - FlapKickTransformerInitializer, - FlipKickTransformerInitializer, - FlogTransformerInitializer, - FlopKickTransformerInitializer, - FrobTransformerInitializer, - PitFileDebtCeilingTransformerInitializer, - PitFileIlkTransformerInitializer, - PriceFeedTransformerInitializer, - TendTransformerInitializer, - VatGrabTransformerInitializer, - VatInitTransformerInitializer, - VatMoveTransformerInitializer, - VatHealTransformerInitializer, - VatFoldTransformerInitializer, - VatSlipTransformerInitializer, - VatTollTransformerInitializer, - VatTuneTransformerInitializer, - VatFluxTransformerInitializer, +// `TransformerInitializers` returns a list of functions, that given a db pointer +// will return a `shared.Transformer` +func TransformerInitializers() (initializers []shared.TransformerInitializer) { + for _, transformer := range logNoteTransformers { + initializers = append(initializers, transformer.NewLogNoteTransformer) } + + for _, transformer := range customEventTransformers { + initializers = append(initializers, transformer.NewTransformer) + } + return } diff --git a/pkg/transformers/vat_flux/repository.go b/pkg/transformers/vat_flux/repository.go index d87ffca7..2febe527 100644 --- a/pkg/transformers/vat_flux/repository.go +++ b/pkg/transformers/vat_flux/repository.go @@ -16,7 +16,6 @@ package vat_flux import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository VatFluxRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.VatFluxChecked) } -func (repository VatFluxRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatFluxChecked) -} - func (repository *VatFluxRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_flux/repository_test.go b/pkg/transformers/vat_flux/repository_test.go index 14b3f403..a8b7ab85 100644 --- a/pkg/transformers/vat_flux/repository_test.go +++ b/pkg/transformers/vat_flux/repository_test.go @@ -86,15 +86,6 @@ var _ = Describe("VatFlux Repository", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_flux.VatFluxRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkHeaderChecked", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatFluxChecked, diff --git a/pkg/transformers/vat_fold/repository.go b/pkg/transformers/vat_fold/repository.go index bb14f172..5d831bca 100644 --- a/pkg/transformers/vat_fold/repository.go +++ b/pkg/transformers/vat_fold/repository.go @@ -17,7 +17,6 @@ package vat_fold import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -63,10 +62,6 @@ func (repository VatFoldRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatFoldChecked) } -func (repository VatFoldRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatFoldChecked) -} - func (repository *VatFoldRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_fold/repository_test.go b/pkg/transformers/vat_fold/repository_test.go index a26ff75b..fc551b37 100644 --- a/pkg/transformers/vat_fold/repository_test.go +++ b/pkg/transformers/vat_fold/repository_test.go @@ -84,13 +84,4 @@ var _ = Describe("Vat.fold repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_fold.VatFoldRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_grab/repository.go b/pkg/transformers/vat_grab/repository.go index b826b7c1..a8a274a4 100644 --- a/pkg/transformers/vat_grab/repository.go +++ b/pkg/transformers/vat_grab/repository.go @@ -3,7 +3,6 @@ package vat_grab import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -47,10 +46,6 @@ func (repository VatGrabRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatGrabChecked) } -func (repository VatGrabRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatGrabChecked) -} - func (repository *VatGrabRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_grab/repository_test.go b/pkg/transformers/vat_grab/repository_test.go index 1d6e222c..e5dded85 100644 --- a/pkg/transformers/vat_grab/repository_test.go +++ b/pkg/transformers/vat_grab/repository_test.go @@ -73,13 +73,4 @@ var _ = Describe("Vat grab repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &vatGrabRepository, - RepositoryTwo: &vat_grab.VatGrabRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_heal/repository.go b/pkg/transformers/vat_heal/repository.go index 5050d6d6..aed8fcd0 100644 --- a/pkg/transformers/vat_heal/repository.go +++ b/pkg/transformers/vat_heal/repository.go @@ -17,7 +17,6 @@ package vat_heal import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -61,10 +60,6 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{}) return tx.Commit() } -func (repository VatHealRepository) MissingHeaders(startingBlock, endingBlock int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlock, endingBlock, repository.db, constants.VatHealChecked) -} - func (repository VatHealRepository) MarkHeaderChecked(headerId int64) error { return shared.MarkHeaderChecked(headerId, repository.db, constants.VatHealChecked) } diff --git a/pkg/transformers/vat_heal/repository_test.go b/pkg/transformers/vat_heal/repository_test.go index 9cc668fc..d16bd9df 100644 --- a/pkg/transformers/vat_heal/repository_test.go +++ b/pkg/transformers/vat_heal/repository_test.go @@ -83,15 +83,6 @@ var _ = Describe("VatHeal Repository", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_heal.VatHealRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkCheckedHeader", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatHealChecked, diff --git a/pkg/transformers/vat_init/repository.go b/pkg/transformers/vat_init/repository.go index cee9e44a..14b2d4c8 100644 --- a/pkg/transformers/vat_init/repository.go +++ b/pkg/transformers/vat_init/repository.go @@ -16,7 +16,6 @@ package vat_init import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -63,10 +62,6 @@ func (repository VatInitRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatInitChecked) } -func (repository VatInitRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatInitChecked) -} - func (repository *VatInitRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_init/repository_test.go b/pkg/transformers/vat_init/repository_test.go index 93ca26fe..6e505171 100644 --- a/pkg/transformers/vat_init/repository_test.go +++ b/pkg/transformers/vat_init/repository_test.go @@ -72,15 +72,6 @@ var _ = Describe("Vat init repository", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_init.VatInitRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkHeaderChecked", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatInitChecked, diff --git a/pkg/transformers/vat_move/repository.go b/pkg/transformers/vat_move/repository.go index f47db4d6..2548c2b4 100644 --- a/pkg/transformers/vat_move/repository.go +++ b/pkg/transformers/vat_move/repository.go @@ -16,7 +16,6 @@ package vat_move import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -60,10 +59,6 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{}) return tx.Commit() } -func (repository VatMoveRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatMoveChecked) -} - func (repository VatMoveRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatMoveChecked) } diff --git a/pkg/transformers/vat_move/repository_test.go b/pkg/transformers/vat_move/repository_test.go index b4eb0bcc..179a2ee0 100644 --- a/pkg/transformers/vat_move/repository_test.go +++ b/pkg/transformers/vat_move/repository_test.go @@ -71,15 +71,6 @@ var _ = Describe("Vat Move", func() { }) }) - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_move.VatMoveRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) - Describe("MarkHeaderChecked", func() { inputs := shared_behaviors.MarkedHeaderCheckedBehaviorInputs{ CheckedHeaderColumnName: constants.VatMoveChecked, diff --git a/pkg/transformers/vat_slip/repository.go b/pkg/transformers/vat_slip/repository.go index 1c6b8d06..15f412ca 100644 --- a/pkg/transformers/vat_slip/repository.go +++ b/pkg/transformers/vat_slip/repository.go @@ -2,7 +2,6 @@ package vat_slip import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -48,10 +47,6 @@ func (repository VatSlipRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatSlipChecked) } -func (repository VatSlipRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatSlipChecked) -} - func (repository *VatSlipRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_slip/repository_test.go b/pkg/transformers/vat_slip/repository_test.go index d3f1e64e..23a83150 100644 --- a/pkg/transformers/vat_slip/repository_test.go +++ b/pkg/transformers/vat_slip/repository_test.go @@ -68,13 +68,4 @@ var _ = Describe("Vat slip repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_slip.VatSlipRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_toll/repository.go b/pkg/transformers/vat_toll/repository.go index 39371193..2c9be8a4 100644 --- a/pkg/transformers/vat_toll/repository.go +++ b/pkg/transformers/vat_toll/repository.go @@ -2,7 +2,6 @@ package vat_toll import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -47,10 +46,6 @@ func (repository VatTollRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTollChecked) } -func (repository VatTollRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTollChecked) -} - func (repository *VatTollRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_toll/repository_test.go b/pkg/transformers/vat_toll/repository_test.go index 6b3d21d4..3c2f9f0d 100644 --- a/pkg/transformers/vat_toll/repository_test.go +++ b/pkg/transformers/vat_toll/repository_test.go @@ -68,13 +68,4 @@ var _ = Describe("Vat toll repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_toll.VatTollRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vat_tune/repository.go b/pkg/transformers/vat_tune/repository.go index b7298cce..c9951ce6 100644 --- a/pkg/transformers/vat_tune/repository.go +++ b/pkg/transformers/vat_tune/repository.go @@ -2,7 +2,6 @@ package vat_tune import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -47,10 +46,6 @@ func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked) } -func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked) -} - func (repository *VatTuneRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vat_tune/repository_test.go b/pkg/transformers/vat_tune/repository_test.go index 5214f9e5..1700825b 100644 --- a/pkg/transformers/vat_tune/repository_test.go +++ b/pkg/transformers/vat_tune/repository_test.go @@ -71,13 +71,4 @@ var _ = Describe("Vat tune repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vat_tune.VatTuneRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) }) diff --git a/pkg/transformers/vow_flog/repository.go b/pkg/transformers/vow_flog/repository.go index 0754fd5f..48076802 100644 --- a/pkg/transformers/vow_flog/repository.go +++ b/pkg/transformers/vow_flog/repository.go @@ -16,7 +16,6 @@ package vow_flog import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants" @@ -64,10 +63,6 @@ func (repository VowFlogRepository) MarkHeaderChecked(headerID int64) error { return shared.MarkHeaderChecked(headerID, repository.db, constants.VowFlogChecked) } -func (repository VowFlogRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { - return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VowFlogChecked) -} - func (repository *VowFlogRepository) SetDB(db *postgres.DB) { repository.db = db } diff --git a/pkg/transformers/vow_flog/repository_test.go b/pkg/transformers/vow_flog/repository_test.go index 153b1876..de6b3b94 100644 --- a/pkg/transformers/vow_flog/repository_test.go +++ b/pkg/transformers/vow_flog/repository_test.go @@ -80,13 +80,4 @@ var _ = Describe("Vow flog repository", func() { shared_behaviors.SharedRepositoryMarkHeaderCheckedBehaviors(&inputs) }) - - Describe("MissingHeaders", func() { - inputs := shared_behaviors.MissingHeadersBehaviorInputs{ - Repository: &repository, - RepositoryTwo: &vow_flog.VowFlogRepository{}, - } - - shared_behaviors.SharedRepositoryMissingHeadersBehaviors(&inputs) - }) })