diff --git a/.travis.yml b/.travis.yml index 237dc152..1faa79d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,28 +5,25 @@ go: services: - postgresql addons: - postgresql: "9.6" - + postgresql: '9.6' go_import_path: github.com/vulcanize/vulcanizedb - before_install: - # ginkgo golint dep goose - make installtools - bash ./scripts/install-postgres-10.sh - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list - sudo apt-get update && sudo apt-get install yarn - before_script: - sudo -u postgres createdb vulcanize_private - make migrate NAME=vulcanize_private - cd postgraphile && yarn - script: - yarn test - cd ../ - make test - make integrationtest - notifications: email: false +env: + matrix: + secure: hz6YPkTm59QhOQ2+05K+AWHw0wOoPjz9wqfUKZcuUi+ICdcuClXMt+hVpUDmwrGOCp8B5y8hyxr/iq7P+3MLwz1/2JYxc9S9MT1O0WXU8ZuWZR0LI1e2ZhZCF3E/JWq6c4atxFIEhcv7roBUkbUcRA8cpRdZmEmpSM8JyP0z76CezIG/HeyUdVW34DLjTBNB0TJdOZyfOh0aCvzgXR/kfKaSYNBhJY7j2UK7x0qK0UlQ/n7RHCrtjWoNWpuwl9bw1F5plMHOD9bq0oDG6gs1SFBaybfEMN71Hp0QxhD/u+1tVuHfGooYhzVgxStPSCpSkgQ7vgSZI766ErqPc3B6Wv9K+s5exPLgCykEiLorW6qI8A+mdiPIiIzLBRMcbF/kCo7gFh0kDIYbSTjS5COfjNw/fKsp59upXF4VtCDgVgjAemY6XT4lziZiVQwiK1Oyln8HrIux1aJEWgRGEQpQqwVeCUHClHus5Paf/N0Ci5f9NHh2zbkZvDuUF2uQu6Wc58wXHcVsloyfQibJrH1q2sQRqdiPfN5Y9l0igFzXILFd4itXMyMnSDQh6+GD6V0YY/hGufAs42UymjGYbEwZQP/gn8/bQpilngbmlJ7bB2nva70kXgqhuNiZM5XuFuQMIP3U2Tbm87FHEFUCoKPv2if9Ft74YkAuzVljAMo2YLQ= diff --git a/README.md b/README.md index 5122e0ec..91e5cd99 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ false If you have full rinkeby chaindata you can move it to `rinkeby_vulcanizedb_geth_data` docker volume to skip long wait of sync. ## Running the Tests +- Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full archival node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url) - `createdb vulcanize_private` will create the test db - `make migrate NAME=vulcanize_private` will run the db migrations - `make test` will run the unit tests and skip the integration tests @@ -154,43 +155,91 @@ Contract watchers work with a light or full sync vDB to fetch raw ethereum data A watcher is composed of at least a fetcher and a transformer or set of transformers, where a fetcher is an interface for retrieving raw Ethereum data from some source (e.g. eth_jsonrpc, IPFS) and a transformer is an interface for filtering through that raw Ethereum data to extract, process, and persist data for specific contracts or accounts. -### omniWatcher -The `omniWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. -It also provides some state variable coverage by automating polling of public methods, with some restrictions. +## contractWatcher +The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. +It also provides some state variable coverage by automating polling of public methods, with some restrictions: +1. The method must have 2 or less arguments +1. The method's arguments must all be of type address or bytes32 (hash) +1. The method must return a single value -This command requires a pre-synced (full or light) vulcanizeDB (see above sections) and currently requires the contract ABI be available on etherscan or provided by the user. +This command operates in two modes- `light` and `full`- which require a light or full-synced vulcanizeDB, respectively. -To watch all events of a contract using a light synced vDB: - - Execute `./vulcanizedb omniWatcher --config --contract-address ` +This command requires the contract ABI be available on Etherscan if it is not provided in the config file by the user. -Or if you are using a full synced vDB, change the mode to full: - - Execute `./vulcanizedb omniWatcher --mode full --config --contract-address ` +If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `light` or `full` mode. +Otherwise, when operating in `light` mode, we only need to connect to a full node to fetch event logs. -To watch contracts on a network other than mainnet, use the network flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --network ` +This command takes a config of the form: -To watch events starting at a certain block use the starting block flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --starting-block-number <#>` +```toml + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 -To watch only specified events use the events flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --events --events ` + [client] + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" -To watch events and poll the specified methods with any addresses and hashes emitted by the watched events utilize the methods flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --methods --methods ` + [contract] + network = "" + addresses = [ + "contractAddress1", + "contractAddress2" + ] + [contract.contractAddress1] + abi = 'ABI for contract 1' + startingBlock = 982463 + [contract.contractAddress2] + abi = 'ABI for contract 2' + events = [ + "event1", + "event2" + ] + eventArgs = [ + "arg1", + "arg2" + ] + methods = [ + "method1", + "method2" + ] + methodArgs = [ + "arg1", + "arg2" + ] + startingBlock = 4448566 + piping = true +```` -To watch specified events and poll the specified method with any addresses and hashes emitted by the watched events: - - Execute `./vulcanizedb omniWatcher --config --contract-address --events --events --methods ` +- The `contract` section defines which contracts we want to watch and with which conditions. +- `network` is only necessary if the ABIs are not provided and wish to be fetched from Etherscan. + - Empty or nil string indicates mainnet + - "ropsten", "kovan", and "rinkeby" indicate their respective networks +- `addresses` lists the contract addresses we are watching and is used to load their individual configuration parameters +- `contract.` are the sub-mappings which contain the parameters specific to each contract address + - `abi` is the ABI for the contract; if none is provided the application will attempt to fetch one from Etherscan using the provided address and network + - `events` is the list of events to watch + - If this field is omitted or no events are provided then by defualt ALL events extracted from the ABI will be watched + - If event names are provided then only those events will be watched + - `eventArgs` is the list of arguments to filter events with + - If this field is omitted or no eventArgs are provided then by default watched events are not filtered by their argument values + - If eventArgs are provided then only those events which emit at least one of these values as an argument are watched + - `methods` is the list of methods to poll + - If this is omitted or no methods are provided then by default NO methods are polled + - If method names are provided then those methods will be polled, provided + 1) Method has two or less arguments + 1) Arguments are all of address or hash types + 1) Method returns a single value + - `methodArgs` is the list of arguments to limit polling methods to + - If this field is omitted or no methodArgs are provided then by default methods will be polled with every combination of the appropriately typed values that have been collected from watched events + - If methodArgs are provided then only those values will be used to poll methods + - `startingBlock` is the block we want to begin watching the contract, usually the deployment block of that contract + - `piping` is a boolean flag which indicates whether or not we want to pipe return method values forward as arguments to subsequent method calls -To turn on method piping so that values returned from previous method calls are cached and used as arguments in subsequent method calls: - - Execute `./vulcanizedb omniWatcher --config --piping true --contract-address --events --events --methods ` +At the very minimum, for each contract address an ABI and a starting block number need to be provided (or just the starting block if the ABI can be reliably fetched from Etherscan). +With just this information we will be able to watch all events at the contract, but with no additional filters and no method polling. -To watch all types of events of the contract but only persist the ones that emit one of the filtered-for argument values: - - Execute `./vulcanizedb omniWatcher --config --contract-address --event-args --event-args ` - -To watch all events of the contract but only poll the specified method with specified argument values (if they are emitted from the watched events): - - Execute `./vulcanizedb omniWatcher --config --contract-address --methods --method-args --method-args ` - -#### omniWatcher output +### contractWatcher output Transformed events and polled method results are committed to Postgres in schemas and tables generated according to the contract abi. @@ -198,12 +247,38 @@ Schemas are created for each contract using the naming convention `_< Under this schema, tables are generated for watched events as `_event` and for polled methods as `_method` The 'method' and 'event' identifiers are tacked onto the end of the table names to prevent collisions between methods and events of the same lowercase name -Example: +### contractWatcher example: -Running `./vulcanizedb omniWatcher --config --starting-block-number=5197514 --contract-address=0x8dd5fbce2f6a956c3022ba3663759011dd51e73e --events=Transfer --events=Mint --methods=balanceOf` -watches Transfer and Mint events of the TrueUSD contract and polls its balanceOf method using the addresses we find emitted from those events +Modify `./environments/example.toml` to replace the empty `ipcPath` with a path that points to an ethjson_rpc endpoint (e.g. a local geth node ipc path or an Infura url). +This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node, +remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract. -It produces and populates a schema with three tables: +If you are operating a light sync vDB, run: + + `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light` + +If instead you are operating a full sync vDB and provided an archival node IPC path, run in full mode: + + `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=full` + +This will run the contractWatcher and configures it to watch the contracts specified in the config file. Note that +by default we operate in `light` mode but the flag is included here to demonstrate its use. + +The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E). + +Because the ENS Registry is configured with only an ABI and a starting block, we will watch all events for this contract and poll none of its methods. Note that the ENS Registry is an example +of a contract which does not have its ABI available over Etherscan and must have it included in the config file. + +The TrueUSD contract is configured with two events (`Transfer` and `Mint`) and a single method (`balanceOf`), as such it will watch these two events and use any addresses it collects emitted from them +to poll the `balanceOf` method with those addresses at every block. Note that we do not provide an ABI for TrueUSD as its ABI can be fetched from Etherscan. + +For the ENS contract, it produces and populates a schema with four tables" +`light_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event` +`light_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event` +`light_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event` +`light_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event` + +For the TrusUSD contract, it produces and populates a schema with three tables: `light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event` `light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event` @@ -236,16 +311,18 @@ Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method" | who_ | character varying(66) | | not null | | extended | | | | returned | numeric | | not null | | main | | | -The addition of '_' after table names is to prevent collisions with reserved Postgres words +The addition of '_' after table names is to prevent collisions with reserved Postgres words. -### composeAndExecute +Also notice that the contract address used for the schema name has been down-cased. + +## composeAndExecute The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers. This is accomplished by generating a Go pluggin which allows our `vulcanizedb` binary to link to external transformers, so long as they abide by our standard [interfaces](https://github.com/vulcanize/maker-vulcanizedb/tree/compose_and_execute/libraries/shared/transformer). This command requires Go 1.11+ and [Go plugins](https://golang.org/pkg/plugin/) only work on Unix based systems. -#### Writing custom transformers +### Writing custom transformers Storage Transformers * [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/README.md) * [Example](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/EXAMPLE.md) @@ -254,7 +331,7 @@ Event Transformers * [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/event_docs/libraries/shared/factories/README.md) * [Example](https://github.com/vulcanize/ens_transformers/tree/working) -#### composeAndExecute configuration +### composeAndExecute configuration A .toml config file is specified when executing the command: `./vulcanizedb composeAndExecute --config=./environments/config_name.toml` @@ -269,7 +346,7 @@ The config provides information for composing a set of transformers: port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] home = "github.com/vulcanize/vulcanizedb" @@ -289,7 +366,7 @@ The config provides information for composing a set of transformers: rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_event" + type = "eth_contract" repository = "github.com/account/repo" migrations = "db/migrations" rank = "0" @@ -319,6 +396,9 @@ The config provides information for composing a set of transformers: that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS) - `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go) that fetches event logs from an ETH node + - `eth_contract` indicates the transformer works with the [contract watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/generic_watcher.go) + that is made to work with [contract_watcher pkg](https://github.com/vulcanize/maker-vulcanizedb/tree/staging/pkg/omni) + based transformers which work with either a light or full sync vDB to watch events and poll public methods ([example](https://github.com/vulcanize/ens_transformers/blob/working/transformers/domain_records/transformer.go)) - `migrations` is the relative path from `repository` to the db migrations directory for the transformer - `rank` determines the order that migrations are ran, with lower ranked migrations running first - this is to help isolate any potential conflicts between transformer migrations @@ -350,25 +430,28 @@ type exporter string var Exporter exporter -func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer { - return []interface1.EventTransformerInitializer{ - transformer1.EventTransformerInitializer, - transformer2.EventTransformerInitializer, - transformer3.EventTransformerInitializer, - }, []interface1.StorageTransformerInitializer{ - transformer4.StorageTransformerInitializer, - } +func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer, []interface1.ContractTransformerInitializer { + return []interface1.TransformerInitializer{ + transformer1.TransformerInitializer, + transformer3.TransformerInitializer, + }, []interface1.StorageTransformerInitializer{ + transformer4.StorageTransformerInitializer, + }, []interface1.ContractTransformerInitializer{ + transformer2.TransformerInitializer, + } } ``` -#### Preparing transformer(s) to work as pluggins for composeAndExecute +### Preparing transformers to work as pluggins for composeAndExecute To plug in an external transformer we need to: * Create a [package](https://github.com/vulcanize/ens_transformers/blob/working/transformers/registry/new_owner/initializer/initializer.go) -that exports a variable `EventTransformerInitializer` or `StorageTransformerInitializer` that are of type [EventTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/event_transformer.go#L33) -or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/storage_transformer.go#L31), respectively -* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go#L83) -or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/storage_watcher.go#L58) watcher execution modes +that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33) +or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), +or [ContractTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/transformer/contract_transformer.go#L31), respectively +* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83), +[storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53), +or [contract](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes * Create db migrations to run against vulcanizeDB so that we can store the transformer output * Do not `goose fix` the transformer migrations * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields diff --git a/cmd/compose.go b/cmd/compose.go index 72f6a299..b8098b9b 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -42,7 +42,7 @@ var composeCmd = &cobra.Command{ port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] home = "github.com/vulcanize/vulcanizedb" @@ -62,7 +62,7 @@ var composeCmd = &cobra.Command{ rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_event" + type = "eth_contract" repository = "github.com/account/repo" migrations = "db/migrations" rank = "0" @@ -91,7 +91,9 @@ from it and loaded into and executed over by the appropriate watcher. The type of watcher that the transformer works with is specified using the type variable for each transformer in the config. Currently there are watchers of event data from an eth node (eth_event) and storage data from an eth node -(eth_storage). +(eth_storage), and a more generic interface for accepting contract_watcher pkg +based transformers which can perform both event watching and public method +polling (eth_contract). Transformers of different types can be ran together in the same command using a single config file or in separate command instances using different config files diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index aef951a3..d2abd045 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -42,7 +42,7 @@ var composeAndExecuteCmd = &cobra.Command{ port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] home = "github.com/vulcanize/vulcanizedb" @@ -62,7 +62,7 @@ var composeAndExecuteCmd = &cobra.Command{ rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_event" + type = "eth_contract" repository = "github.com/account/repo" migrations = "db/migrations" rank = "2" @@ -91,7 +91,9 @@ from it and loaded into and executed over by the appropriate watcher. The type of watcher that the transformer works with is specified using the type variable for each transformer in the config. Currently there are watchers of event data from an eth node (eth_event) and storage data from an eth node -(eth_storage). +(eth_storage), and a more generic interface for accepting contract_watcher pkg +based transformers which can perform both event watching and public method +polling (eth_contract). Transformers of different types can be ran together in the same command using a single config file or in separate command instances using different config files @@ -149,8 +151,8 @@ func composeAndExecute() { os.Exit(1) } - // Use the Exporters export method to load the EventTransformerInitializer and StorageTransformerInitializer sets - ethEventInitializers, ethStorageInitializers := exporter.Export() + // Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets + ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export() // Setup bc and db objects blockChain := getBlockChain() @@ -173,6 +175,13 @@ func composeAndExecute() { wg.Add(1) go watchEthStorage(&sw, &wg) } + + if len(ethContractInitializers) > 0 { + gw := watcher.NewContractWatcher(&db, blockChain) + gw.AddTransformers(ethContractInitializers) + wg.Add(1) + go watchEthContract(&gw, &wg) + } wg.Wait() } diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go new file mode 100644 index 00000000..d27802ce --- /dev/null +++ b/cmd/contractWatcher.go @@ -0,0 +1,125 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" + lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" + "github.com/vulcanize/vulcanizedb/utils" +) + +// contractWatcherCmd represents the contractWatcher command +var contractWatcherCmd = &cobra.Command{ + Use: "contractWatcher", + Short: "Watches events at the provided contract address using fully synced vDB", + Long: `Uses input contract address and event filters to watch events + +Expects an ethereum node to be running +Expects an archival node synced into vulcanizeDB +Requires a .toml config file: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" + + [contract] + network = "" + addresses = [ + "contractAddress1", + "contractAddress2" + ] + [contract.contractAddress1] + abi = 'ABI for contract 1' + startingBlock = 982463 + [contract.contractAddress2] + abi = 'ABI for contract 2' + events = [ + "event1", + "event2" + ] + eventArgs = [ + "arg1", + "arg2" + ] + methods = [ + "method1", + "method2" + ] + methodArgs = [ + "arg1", + "arg2" + ] + startingBlock = 4448566 + piping = true +`, + Run: func(cmd *cobra.Command, args []string) { + contractWatcher() + }, +} + +var ( + mode string +) + +func contractWatcher() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + blockChain := getBlockChain() + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + + var t st.ContractTransformer + con := config.ContractConfig{} + con.PrepConfig() + switch mode { + case "light": + t = lt.NewTransformer(con, blockChain, &db) + case "full": + t = ft.NewTransformer(con, blockChain, &db) + default: + log.Fatal("Invalid mode") + } + + err := t.Init() + if err != nil { + log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) + } + + for range ticker.C { + err = t.Execute() + if err != nil { + log.Error("Execution error for transformer:", t.GetConfig().Name, err) + } + } +} + +func init() { + rootCmd.AddCommand(contractWatcherCmd) + contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") +} diff --git a/cmd/execute.go b/cmd/execute.go index da4bea4e..3ebc15f1 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -46,7 +46,7 @@ var executeCmd = &cobra.Command{ port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] name = "exampleTransformerExporter" @@ -99,8 +99,8 @@ func execute() { os.Exit(1) } - // Use the Exporters export method to load the EventTransformerInitializer and StorageTransformerInitializer sets - ethEventInitializers, ethStorageInitializers := exporter.Export() + // Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets + ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export() // Setup bc and db objects blockChain := getBlockChain() @@ -123,6 +123,13 @@ func execute() { wg.Add(1) go watchEthStorage(&sw, &wg) } + + if len(ethContractInitializers) > 0 { + gw := watcher.NewContractWatcher(&db, blockChain) + gw.AddTransformers(ethContractInitializers) + wg.Add(1) + go watchEthContract(&gw, &wg) + } wg.Wait() } @@ -132,7 +139,7 @@ func init() { } type Exporter interface { - Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer) + Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.ContractTransformerInitializer) } func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { @@ -148,23 +155,28 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { - err := w.Execute(recheck) - if err != nil { - // TODO Handle watcher errors in execute - } + w.Execute(recheck) } } func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { defer wg.Done() - // Execute over the StorageTransformerInitializer set using the watcher + // Execute over the StorageTransformerInitializer set using the storage watcher log.Info("executing storage transformers") ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { - err := w.Execute() - if err != nil { - // TODO Handle watcher errors in execute - } + w.Execute() + } +} + +func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) { + defer wg.Done() + // Execute over the ContractTransformerInitializer set using the contract watcher + log.Info("executing contract_watcher transformers") + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() + for range ticker.C { + w.Execute() } } diff --git a/cmd/omniWatcher.go b/cmd/omniWatcher.go deleted file mode 100644 index a07108a9..00000000 --- a/cmd/omniWatcher.go +++ /dev/null @@ -1,121 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package cmd - -import ( - "fmt" - "log" - "time" - - "github.com/spf13/cobra" - - ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - st "github.com/vulcanize/vulcanizedb/pkg/omni/shared/transformer" - "github.com/vulcanize/vulcanizedb/utils" -) - -// omniWatcherCmd represents the omniWatcher command -var omniWatcherCmd = &cobra.Command{ - Use: "omniWatcher", - Short: "Watches events at the provided contract address using fully synced vDB", - Long: `Uses input contract address and event filters to watch events - -Expects an ethereum node to be running -Expects an archival node synced into vulcanizeDB -Requires a .toml config file: - - [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - - [client] - ipcPath = "/Users/user/Library/Ethereum/geth.ipc" -`, - Run: func(cmd *cobra.Command, args []string) { - omniWatcher() - }, -} - -var ( - network string - contractAddress string - contractAddresses []string - contractEvents []string - contractMethods []string - eventArgs []string - methodArgs []string - methodPiping bool - mode string -) - -func omniWatcher() { - if contractAddress == "" && len(contractAddresses) == 0 { - log.Fatal("Contract address required") - } - - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - blockChain := getBlockChain() - db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - - var t st.Transformer - switch mode { - case "light": - t = lt.NewTransformer(network, blockChain, &db) - case "full": - t = ft.NewTransformer(network, blockChain, &db) - default: - log.Fatal("Invalid mode") - } - - contractAddresses = append(contractAddresses, contractAddress) - for _, addr := range contractAddresses { - t.SetEvents(addr, contractEvents) - t.SetMethods(addr, contractMethods) - t.SetEventArgs(addr, eventArgs) - t.SetMethodArgs(addr, methodArgs) - t.SetPiping(addr, methodPiping) - t.SetStartingBlock(addr, startingBlockNumber) - } - - err := t.Init() - if err != nil { - log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) - } - - for range ticker.C { - t.Execute() - } -} - -func init() { - rootCmd.AddCommand(omniWatcherCmd) - - omniWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") - omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") - omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address") - omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched") - omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled") - omniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified") - omniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here") - omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) - omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists") - omniWatcherCmd.Flags().BoolVarP(&methodPiping, "piping", "p", false, "Turn on method output piping: methods listed first will be polled first and their output used as input to subsequent methods") -} diff --git a/environments/example.toml b/environments/example.toml new file mode 100644 index 00000000..e025baaa --- /dev/null +++ b/environments/example.toml @@ -0,0 +1,26 @@ +[database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + +[client] + ipcPath = "" + +[contract] + network = "" + addresses = [ + "0x314159265dD8dbb310642f98f50C066173C1259b", + "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E" + ] + [contract.0x314159265dD8dbb310642f98f50C066173C1259b] + abi = '[{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"resolver","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"label","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setSubnodeOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"ttl","type":"uint64"}],"name":"setTTL","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"ttl","outputs":[{"name":"","type":"uint64"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"resolver","type":"address"}],"name":"setResolver","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"label","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"NewOwner","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"resolver","type":"address"}],"name":"NewResolver","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"ttl","type":"uint64"}],"name":"NewTTL","type":"event"}]' + startingBlock = 3327417 + [contract.0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E] + events = [ + "Transfer", + "Issue" + ] + methods = [ + "balanceOf" + ] + startingBlock = 5197514 \ No newline at end of file diff --git a/environments/infura.toml b/environments/infura.toml index 095e747d..fb9876fb 100644 --- a/environments/infura.toml +++ b/environments/infura.toml @@ -1,7 +1,7 @@ [database] -name = "vulcanize_public" -hostname = "localhost" -port = 5432 + name = "vulcanize_public" + hostname = "localhost" + port = 5432 [client] -ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" \ No newline at end of file + ipcPath = "" diff --git a/environments/private.toml b/environments/private.toml index 30382733..83f80f7b 100644 --- a/environments/private.toml +++ b/environments/private.toml @@ -1,7 +1,7 @@ [database] -name = "vulcanize_private" -hostname = "localhost" -port = 5432 + name = "vulcanize_private" + hostname = "localhost" + port = 5432 [client] -ipcPath = "http://127.0.0.1:7545" \ No newline at end of file + ipcPath = "http://127.0.0.1:7545" diff --git a/environments/public.toml.example b/environments/public.toml.example index 1e748bfb..52f8a2d1 100644 --- a/environments/public.toml.example +++ b/environments/public.toml.example @@ -1,8 +1,8 @@ [database] -name = "vulcanize_public" -hostname = "localhost" -port = 5432 + name = "vulcanize_public" + hostname = "localhost" + port = 5432 [client] -ipcPath = -levelDbPath = \ No newline at end of file + ipcPath = + levelDbPath = diff --git a/integration_test/omni_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go similarity index 72% rename from integration_test/omni_full_transformer_test.go rename to integration_test/contract_watcher_full_transformer_test.go index 5346584d..bcb7a29a 100644 --- a/integration_test/omni_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -2,6 +2,7 @@ package integration import ( "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "math/rand" "strings" "time" @@ -10,16 +11,16 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) -var _ = Describe("Omni full transformer", func() { +var _ = Describe("contractWatcher full transformer", func() { var db *postgres.DB var err error var blockChain core.BlockChain @@ -41,8 +42,7 @@ var _ = Describe("Omni full transformer", func() { It("Initializes transformer's contract objects", func() { blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -50,11 +50,32 @@ var _ = Describe("Omni full transformer", func() { Expect(ok).To(Equal(true)) Expect(c.StartingBlock).To(Equal(int64(6194633))) - Expect(c.LastBlock).To(Equal(int64(6194634))) Expect(c.Abi).To(Equal(constants.TusdAbiString)) Expect(c.Name).To(Equal("TrueUSD")) Expect(c.Address).To(Equal(tusdAddr)) }) + + It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) + }) + + It("Does nothing if watched events are unset", func() { + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Events = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no filters created")) + + _, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(false)) + }) }) Describe("Execute", func() { @@ -64,9 +85,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -85,9 +104,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -119,9 +141,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -142,15 +167,15 @@ var _ = Describe("Omni full transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("transformer has no initialized contracts to work with")) }) }) @@ -161,9 +186,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) + t := transformer.NewTransformer(test_helpers.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -183,9 +206,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -210,9 +236,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -232,13 +261,16 @@ var _ = Describe("Omni full transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("It does not perist events if they do not pass the emitted arg filter", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -249,13 +281,19 @@ var _ = Describe("Omni full transformer", func() { log := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not exist")) }) It("If a method arg filter is applied, only those arguments are used in polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, + } + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -270,6 +308,7 @@ var _ = Describe("Omni full transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) }) }) diff --git a/integration_test/omni_light_transformer_test.go b/integration_test/contract_watcher_light_transformer_test.go similarity index 71% rename from integration_test/omni_light_transformer_test.go rename to integration_test/contract_watcher_light_transformer_test.go index 5634ba14..cf9efa77 100644 --- a/integration_test/omni_light_transformer_test.go +++ b/integration_test/contract_watcher_light_transformer_test.go @@ -8,21 +8,22 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) -var _ = Describe("Omnit light transformer", func() { +var _ = Describe("contractWatcher light transformer", func() { var db *postgres.DB var err error var blockChain core.BlockChain var headerRepository repositories.HeaderRepository - var headerID, headerID2 int64 + var headerID int64 var ensAddr = strings.ToLower(constants.EnsContractAddress) var tusdAddr = strings.ToLower(constants.TusdContractAddress) @@ -39,8 +40,7 @@ var _ = Describe("Omnit light transformer", func() { It("Initializes transformer's contract objects", func() { headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -48,11 +48,31 @@ var _ = Describe("Omnit light transformer", func() { Expect(ok).To(Equal(true)) Expect(c.StartingBlock).To(Equal(int64(6194632))) - Expect(c.LastBlock).To(Equal(int64(-1))) Expect(c.Abi).To(Equal(constants.TusdAbiString)) Expect(c.Name).To(Equal("TrueUSD")) Expect(c.Address).To(Equal(tusdAddr)) }) + + It("Fails to initialize if first and block cannot be fetched from vDB headers table", func() { + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) + }) + + It("Does nothing if nothing if no addresses are configured", func() { + headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Addresses = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + _, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(false)) + }) }) Describe("Execute- against TrueUSD contract", func() { @@ -70,9 +90,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -89,9 +107,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) c, ok := t.Contracts[tusdAddr] @@ -131,9 +152,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -147,14 +171,14 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("transformer has no initialized contracts")) }) }) @@ -173,13 +197,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) + t := transformer.NewTransformer(test_helpers.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() Expect(err).ToNot(HaveOccurred()) + Expect(t.Start).To(Equal(int64(6885698))) log := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) @@ -192,9 +215,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) c, ok := t.Contracts[ensAddr] @@ -218,9 +244,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given method using list of collected hashes", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -239,13 +268,16 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("It does not persist events if they do not pass the emitted arg filter", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -254,13 +286,19 @@ var _ = Describe("Omnit light transformer", func() { log := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not exist")) }) It("If a method arg filter is applied, only those arguments are used in polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSConfig + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, + } + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -274,49 +312,32 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) }) Describe("Execute- against both ENS and TrueUSD", func() { BeforeEach(func() { - header1, err := blockChain.GetHeaderByNumber(6791668) - Expect(err).ToNot(HaveOccurred()) - header2, err := blockChain.GetHeaderByNumber(6791669) - Expect(err).ToNot(HaveOccurred()) - header3, err := blockChain.GetHeaderByNumber(6791670) - Expect(err).ToNot(HaveOccurred()) - header4, err := blockChain.GetHeaderByNumber(6885695) - Expect(err).ToNot(HaveOccurred()) - header5, err := blockChain.GetHeaderByNumber(6885696) - Expect(err).ToNot(HaveOccurred()) - header6, err := blockChain.GetHeaderByNumber(6885697) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header1) - headerID, err = headerRepository.CreateOrUpdateHeader(header2) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header3) - headerRepository.CreateOrUpdateHeader(header4) - headerID2, err = headerRepository.CreateOrUpdateHeader(header5) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header6) + for i := 6885692; i <= 6885701; i++ { + header, err := blockChain.GetHeaderByNumber(int64(i)) + Expect(err).ToNot(HaveOccurred()) + _, err = headerRepository.CreateOrUpdateHeader(header) + Expect(err).ToNot(HaveOccurred()) + } }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() Expect(err).ToNot(HaveOccurred()) + Expect(t.Start).To(Equal(int64(6885702))) newOwnerLog := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) Expect(err).ToNot(HaveOccurred()) // We don't know vulcID, so compare individual fields instead of complete structures - Expect(newOwnerLog.HeaderID).To(Equal(headerID2)) Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) @@ -325,18 +346,19 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog) Expect(err).ToNot(HaveOccurred()) // We don't know vulcID, so compare individual fields instead of complete structures - Expect(transferLog.HeaderID).To(Equal(headerID)) - Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) - Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) - Expect(transferLog.Value).To(Equal("9998940000000000000000")) + Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")) + Expect(transferLog.To).To(Equal("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")) + Expect(transferLog.Value).To(Equal("2800000000000000000000")) }) It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) ens, ok := t.Contracts[ensAddr] @@ -347,7 +369,7 @@ var _ = Describe("Omnit light transformer", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(ens.EmittedHashes)).To(Equal(2)) Expect(len(ens.EmittedAddrs)).To(Equal(0)) - Expect(len(tusd.EmittedAddrs)).To(Equal(4)) + Expect(len(tusd.EmittedAddrs)).To(Equal(2)) Expect(len(tusd.EmittedHashes)).To(Equal(0)) b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] @@ -358,29 +380,26 @@ var _ = Describe("Omnit light transformer", func() { Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + b, ok = tusd.EmittedAddrs[common.HexToAddress("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) + _, ok = tusd.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] + Expect(ok).To(Equal(false)) }) It("Polls given methods for each contract, using list of collected values", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = test_helpers.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -399,15 +418,17 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) bal := test_helpers.BalanceOf{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).ToNot(HaveOccurred()) - Expect(bal.Balance).To(Equal("55849938025000000000000")) + Expect(bal.Balance).To(Equal("1954436000000000000000")) Expect(bal.TokenName).To(Equal("TrueUSD")) - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) }) }) diff --git a/pkg/omni/shared/poller/poller_test.go b/integration_test/poller_test.go similarity index 89% rename from pkg/omni/shared/poller/poller_test.go rename to integration_test/poller_test.go index 7b2df18d..fe913176 100644 --- a/pkg/omni/shared/poller/poller_test.go +++ b/integration_test/poller_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package poller_test +package integration_test import ( "fmt" @@ -23,18 +23,18 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Poller", func() { - var p poller.Poller + var contractPoller poller.Poller var con *contract.Contract var db *postgres.DB var bc core.BlockChain @@ -46,7 +46,7 @@ var _ = Describe("Poller", func() { Describe("Full sync mode", func() { BeforeEach(func() { db, bc = test_helpers.SetupDBandBC() - p = poller.NewPoller(bc, db, types.FullSync) + contractPoller = poller.NewPoller(bc, db, types.FullSync) }) Describe("PollContract", func() { @@ -54,10 +54,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, []string{"balanceOf"}) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -89,7 +88,7 @@ var _ = Describe("Poller", func() { Expect(len(con.Methods)).To(Equal(1)) con.AddEmittedHash(common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"), common.HexToHash("0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86")) - err := p.PollContractAt(*con, 6885877) + err := contractPoller.PollContractAt(*con, 6885877) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.Owner{} @@ -109,10 +108,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, nil) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -125,7 +123,7 @@ var _ = Describe("Poller", func() { Describe("FetchContractData", func() { It("Calls a single contract method", func() { var name = new(string) - err := p.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) + err := contractPoller.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) Expect(err).ToNot(HaveOccurred()) Expect(*name).To(Equal("TrueUSD")) }) @@ -135,7 +133,7 @@ var _ = Describe("Poller", func() { Describe("Light sync mode", func() { BeforeEach(func() { db, bc = test_helpers.SetupDBandBC() - p = poller.NewPoller(bc, db, types.LightSync) + contractPoller = poller.NewPoller(bc, db, types.LightSync) }) Describe("PollContract", func() { @@ -143,10 +141,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, []string{"balanceOf"}) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -178,7 +175,7 @@ var _ = Describe("Poller", func() { Expect(len(con.Methods)).To(Equal(1)) con.AddEmittedHash(common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"), common.HexToHash("0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86")) - err := p.PollContractAt(*con, 6885877) + err := contractPoller.PollContractAt(*con, 6885877) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.Owner{} @@ -198,10 +195,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, nil) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -214,11 +210,10 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupENSContract(nil, []string{"resolver"}) Expect(con.Abi).To(Equal(constants.ENSAbiString)) con.StartingBlock = 6921967 - con.LastBlock = 6921968 con.EmittedAddrs = map[interface{}]bool{} con.Piping = false con.AddEmittedHash(common.HexToHash("0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6921968) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.Resolver{} @@ -230,10 +225,10 @@ var _ = Describe("Poller", func() { test_helpers.TearDown(db) db, bc = test_helpers.SetupDBandBC() - p = poller.NewPoller(bc, db, types.LightSync) + contractPoller = poller.NewPoller(bc, db, types.LightSync) con.Piping = true - err = p.PollContract(*con) + err = contractPoller.PollContract(*con, 6921968) Expect(err).ToNot(HaveOccurred()) err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct) @@ -248,7 +243,7 @@ var _ = Describe("Poller", func() { Describe("FetchContractData", func() { It("Calls a single contract method", func() { var name = new(string) - err := p.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) + err := contractPoller.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) Expect(err).ToNot(HaveOccurred()) Expect(*name).To(Equal("TrueUSD")) }) diff --git a/libraries/shared/factories/event/README.md b/libraries/shared/factories/event/README.md index 0a1aa33c..9ad2093a 100644 --- a/libraries/shared/factories/event/README.md +++ b/libraries/shared/factories/event/README.md @@ -264,7 +264,7 @@ func (repository *ExampleRepository) SetDB(db *postgres.DB) { } func (repository ExampleRepository) Create(headerID int64, models []interface{}) error { - tx, dBaseErr := repository.db.Begin() + tx, dBaseErr := repository.db.Beginx() if dBaseErr != nil { return dBaseErr } diff --git a/libraries/shared/repository/repository_test.go b/libraries/shared/repository/repository_test.go index 1a9977d0..ccf6257f 100644 --- a/libraries/shared/repository/repository_test.go +++ b/libraries/shared/repository/repository_test.go @@ -26,12 +26,12 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/constants" shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository" + r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" - r2 "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" "github.com/vulcanize/vulcanizedb/test_config" ) diff --git a/pkg/omni/shared/poller/poller_suite_test.go b/libraries/shared/transformer/contract_transformer.go similarity index 66% rename from pkg/omni/shared/poller/poller_suite_test.go rename to libraries/shared/transformer/contract_transformer.go index 9b4e94b5..14b8ab97 100644 --- a/pkg/omni/shared/poller/poller_suite_test.go +++ b/libraries/shared/transformer/contract_transformer.go @@ -14,22 +14,18 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package poller_test +package transformer import ( - "github.com/sirupsen/logrus" - "io/ioutil" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -func TestPoller(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Poller Suite Test") +type ContractTransformer interface { + Init() error + Execute() error + GetConfig() config.ContractConfig } -var _ = BeforeSuite(func() { - logrus.SetOutput(ioutil.Discard) -}) +type ContractTransformerInitializer func(db *postgres.DB, bc core.BlockChain) ContractTransformer diff --git a/libraries/shared/watcher/contract_watcher.go b/libraries/shared/watcher/contract_watcher.go new file mode 100644 index 00000000..ebb4cc37 --- /dev/null +++ b/libraries/shared/watcher/contract_watcher.go @@ -0,0 +1,76 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Contract watcher is built with a more generic interface +// that allows offloading more of the operational logic to +// the transformers, allowing them to act more dynamically +// Built to work primarily with the contract_watcher packaging +package watcher + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type ContractWatcher struct { + Transformers []transformer.ContractTransformer + DB *postgres.DB + BlockChain core.BlockChain +} + +func NewContractWatcher(db *postgres.DB, bc core.BlockChain) ContractWatcher { + return ContractWatcher{ + DB: db, + BlockChain: bc, + } +} + +func (watcher *ContractWatcher) AddTransformers(inits interface{}) error { + initializers, ok := inits.([]transformer.ContractTransformerInitializer) + if !ok { + return fmt.Errorf("initializers of type %T, not %T", inits, []transformer.ContractTransformerInitializer{}) + } + + for _, initializer := range initializers { + t := initializer(watcher.DB, watcher.BlockChain) + watcher.Transformers = append(watcher.Transformers, t) + } + + for _, transformer := range watcher.Transformers { + err := transformer.Init() + if err != nil { + log.Print("Unable to initialize transformer:", transformer.GetConfig().Name, err) + return err + } + } + return nil +} + +func (watcher *ContractWatcher) Execute() error { + for _, transformer := range watcher.Transformers { + err := transformer.Execute() + if err != nil { + log.Error("Unable to execute transformer:", transformer.GetConfig().Name, err) + return err + } + } + return nil +} diff --git a/pkg/config/contract.go b/pkg/config/contract.go new file mode 100644 index 00000000..50b8366a --- /dev/null +++ b/pkg/config/contract.go @@ -0,0 +1,217 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "strings" +) + +// Config struct for generic contract transformer +type ContractConfig struct { + // Name for the transformer + Name string + + // Ethereum network name; default "" is mainnet + Network string + + // List of contract addresses (map to ensure no duplicates) + Addresses map[string]bool + + // Map of contract address to abi + // If an address has no associated abi the parser will attempt to fetch one from etherscan + Abis map[string]string + + // Map of contract address to slice of events + // Used to set which addresses to watch + // If any events are listed in the slice only those will be watched + // Otherwise all events in the contract ABI are watched + Events map[string][]string + + // Map of contract address to slice of methods + // If any methods are listed in the slice only those will be polled + // Otherwise no methods will be polled + Methods map[string][]string + + // Map of contract address to slice of event arguments to filter for + // If arguments are provided then only events which emit those arguments are watched + // Otherwise arguments are not filtered on events + EventArgs map[string][]string + + // Map of contract address to slice of method arguments to limit polling to + // If arguments are provided then only those arguments are allowed as arguments in method polling + // Otherwise any argument of the right type seen emitted from events at that contract will be used in method polling + MethodArgs map[string][]string + + // Map of contract address to their starting block + StartingBlocks map[string]int64 + + // Map of contract address to whether or not to pipe method polling results forward into subsequent method calls + Piping map[string]bool +} + +func (contractConfig *ContractConfig) PrepConfig() { + addrs := viper.GetStringSlice("contract.addresses") + contractConfig.Network = viper.GetString("contract.network") + contractConfig.Addresses = make(map[string]bool, len(addrs)) + contractConfig.Abis = make(map[string]string, len(addrs)) + contractConfig.Methods = make(map[string][]string, len(addrs)) + contractConfig.Events = make(map[string][]string, len(addrs)) + contractConfig.MethodArgs = make(map[string][]string, len(addrs)) + contractConfig.EventArgs = make(map[string][]string, len(addrs)) + contractConfig.StartingBlocks = make(map[string]int64, len(addrs)) + contractConfig.Piping = make(map[string]bool, len(addrs)) + // De-dupe addresses + for _, addr := range addrs { + contractConfig.Addresses[strings.ToLower(addr)] = true + } + + // Iterate over addresses to pull out config info for each contract + for _, addr := range addrs { + transformer := viper.GetStringMap("contract." + addr) + + // Get and check abi + var abi string + abiInterface, abiOK := transformer["abi"] + if !abiOK { + log.Warnf("contract %s not configured with an ABI, will attempt to fetch it from Etherscan\r\n", addr) + } else { + abi, abiOK = abiInterface.(string) + if !abiOK { + log.Fatal(addr, "transformer `abi` not of type []string") + } + } + if abi != "" { + if _, abiErr := geth.ParseAbi(abi); abiErr != nil { + log.Fatal(addr, "transformer `abi` not valid JSON") + } + } + contractConfig.Abis[strings.ToLower(addr)] = abi + + // Get and check events + events := make([]string, 0) + eventsInterface, eventsOK := transformer["events"] + if !eventsOK { + log.Warnf("contract %s not configured with a list of events to watch, will watch all events\r\n", addr) + events = []string{} + } else { + eventsI, eventsOK := eventsInterface.([]interface{}) + if !eventsOK { + log.Fatal(addr, "transformer `events` not of type []string\r\n") + } + for _, strI := range eventsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `events` not of type []string\r\n") + } + events = append(events, str) + } + } + contractConfig.Events[strings.ToLower(addr)] = events + + // Get and check methods + methods := make([]string, 0) + methodsInterface, methodsOK := transformer["methods"] + if !methodsOK { + log.Warnf("contract %s not configured with a list of methods to poll, will not poll any methods\r\n", addr) + methods = []string{} + } else { + methodsI, methodsOK := methodsInterface.([]interface{}) + if !methodsOK { + log.Fatal(addr, "transformer `methods` not of type []string\r\n") + } + for _, strI := range methodsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `methods` not of type []string\r\n") + } + methods = append(methods, str) + } + } + contractConfig.Methods[strings.ToLower(addr)] = methods + + // Get and check eventArgs + eventArgs := make([]string, 0) + eventArgsInterface, eventArgsOK := transformer["eventArgs"] + if !eventArgsOK { + log.Warnf("contract %s not configured with a list of event arguments to filter for, will not filter events for specific emitted values\r\n", addr) + eventArgs = []string{} + } else { + eventArgsI, eventArgsOK := eventArgsInterface.([]interface{}) + if !eventArgsOK { + log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n") + } + for _, strI := range eventArgsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n") + } + eventArgs = append(eventArgs, str) + } + } + contractConfig.EventArgs[strings.ToLower(addr)] = eventArgs + + // Get and check methodArgs + methodArgs := make([]string, 0) + methodArgsInterface, methodArgsOK := transformer["methodArgs"] + if !methodArgsOK { + log.Warnf("contract %s not configured with a list of method argument values to poll with, will poll methods with all available arguments\r\n", addr) + methodArgs = []string{} + } else { + methodArgsI, methodArgsOK := methodArgsInterface.([]interface{}) + if !methodArgsOK { + log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n") + } + for _, strI := range methodArgsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n") + } + methodArgs = append(methodArgs, str) + } + } + contractConfig.MethodArgs[strings.ToLower(addr)] = methodArgs + + // Get and check startingBlock + startInterface, startOK := transformer["startingblock"] + if !startOK { + log.Fatal(addr, "transformer config is missing `startingBlock` value\r\n") + } + start, startOK := startInterface.(int64) + if !startOK { + log.Fatal(addr, "transformer `startingBlock` not of type int\r\n") + } + contractConfig.StartingBlocks[strings.ToLower(addr)] = start + + // Get pipping + var piping bool + _, pipeOK := transformer["piping"] + if !pipeOK { + log.Warnf("contract %s does not have its `piping` set, by default piping is turned off\r\n", addr) + piping = false + } else { + pipingInterface := transformer["piping"] + piping, pipeOK = pipingInterface.(bool) + if !pipeOK { + log.Fatal(addr, "transformer `piping` not of type bool\r\n") + } + } + contractConfig.Piping[strings.ToLower(addr)] = piping + } +} diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index dbe6a2f3..fad07cbc 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -41,13 +41,13 @@ type Transformer struct { RepositoryPath string } -func (c *Plugin) GetPluginPaths() (string, string, error) { - path, err := helpers.CleanPath(c.FilePath) +func (pluginConfig *Plugin) GetPluginPaths() (string, string, error) { + path, err := helpers.CleanPath(pluginConfig.FilePath) if err != nil { return "", "", err } - name := strings.Split(c.FileName, ".")[0] + name := strings.Split(pluginConfig.FileName, ".")[0] goFile := filepath.Join(path, name+".go") soFile := filepath.Join(path, name+".so") @@ -55,13 +55,13 @@ func (c *Plugin) GetPluginPaths() (string, string, error) { } // Removes duplicate migration paths and returns them in ranked order -func (c *Plugin) GetMigrationsPaths() ([]string, error) { +func (pluginConfig *Plugin) GetMigrationsPaths() ([]string, error) { paths := make(map[uint64]string) highestRank := -1 - for name, transformer := range c.Transformers { + for name, transformer := range pluginConfig.Transformers { repo := transformer.RepositoryPath mig := transformer.MigrationPath - path := filepath.Join("$GOPATH/src", c.Home, "vendor", repo, mig) + path := filepath.Join("$GOPATH/src", pluginConfig.Home, "vendor", repo, mig) cleanPath, err := helpers.CleanPath(path) if err != nil { return nil, err @@ -96,9 +96,9 @@ func (c *Plugin) GetMigrationsPaths() ([]string, error) { } // Removes duplicate repo paths before returning them -func (c *Plugin) GetRepoPaths() map[string]bool { +func (pluginConfig *Plugin) GetRepoPaths() map[string]bool { paths := make(map[string]bool) - for _, transformer := range c.Transformers { + for _, transformer := range pluginConfig.Transformers { paths[transformer.RepositoryPath] = true } @@ -111,26 +111,29 @@ const ( UnknownTransformerType TransformerType = iota EthEvent EthStorage + EthContract ) -func (pt TransformerType) String() string { +func (transformerType TransformerType) String() string { names := [...]string{ "Unknown", "eth_event", "eth_storage", + "eth_contract", } - if pt > EthStorage || pt < EthEvent { + if transformerType > EthContract || transformerType < EthEvent { return "Unknown" } - return names[pt] + return names[transformerType] } func GetTransformerType(str string) TransformerType { types := [...]TransformerType{ EthEvent, EthStorage, + EthContract, } for _, ty := range types { diff --git a/pkg/omni/full/converter/converter.go b/pkg/contract_watcher/full/converter/converter.go similarity index 88% rename from pkg/omni/full/converter/converter.go rename to pkg/contract_watcher/full/converter/converter.go index 2810438a..09bae256 100644 --- a/pkg/omni/full/converter/converter.go +++ b/pkg/contract_watcher/full/converter/converter.go @@ -26,35 +26,29 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Converter is used to convert watched event logs to // custom logs containing event input name => value maps -type Converter interface { +type ConverterInterface interface { Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) Update(info *contract.Contract) } -type converter struct { +type Converter struct { ContractInfo *contract.Contract } -func NewConverter(info *contract.Contract) *converter { - return &converter{ - ContractInfo: info, - } -} - -func (c *converter) Update(info *contract.Contract) { +func (c *Converter) Update(info *contract.Contract) { c.ContractInfo = info } // Convert the given watched event log into a types.Log for the given event -func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { +func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) log := helpers.ConvertToLog(watchedEvent) diff --git a/pkg/omni/full/converter/converter_suite_test.go b/pkg/contract_watcher/full/converter/converter_suite_test.go similarity index 100% rename from pkg/omni/full/converter/converter_suite_test.go rename to pkg/contract_watcher/full/converter/converter_suite_test.go diff --git a/pkg/omni/full/converter/converter_test.go b/pkg/contract_watcher/full/converter/converter_test.go similarity index 84% rename from pkg/omni/full/converter/converter_test.go rename to pkg/contract_watcher/full/converter/converter_test.go index 8fa7b8f5..864a53a2 100644 --- a/pkg/omni/full/converter/converter_test.go +++ b/pkg/contract_watcher/full/converter/converter_test.go @@ -21,11 +21,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" ) var _ = Describe("Converter", func() { @@ -39,7 +39,8 @@ var _ = Describe("Converter", func() { Describe("Update", func() { It("Updates contract con held by the converter", func() { - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) Expect(c.ContractInfo).To(Equal(con)) con := test_helpers.SetupTusdContract([]string{}, []string{}) @@ -58,7 +59,8 @@ var _ = Describe("Converter", func() { err = con.GenerateFilters() Expect(err).ToNot(HaveOccurred()) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) log, err := c.Convert(mocks.MockTranferEvent, event) Expect(err).ToNot(HaveOccurred()) @@ -77,7 +79,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["Transfer"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) _, err := c.Convert(mocks.MockTranferEvent, event) Expect(err).ToNot(HaveOccurred()) @@ -101,7 +104,8 @@ var _ = Describe("Converter", func() { It("Fails with an empty contract", func() { event := con.Events["Transfer"] - c := converter.NewConverter(&contract.Contract{}) + c := converter.Converter{} + c.Update(&contract.Contract{}) _, err = c.Convert(mocks.MockTranferEvent, event) Expect(err).To(HaveOccurred()) }) diff --git a/pkg/omni/full/retriever/block_retriever.go b/pkg/contract_watcher/full/retriever/block_retriever.go similarity index 100% rename from pkg/omni/full/retriever/block_retriever.go rename to pkg/contract_watcher/full/retriever/block_retriever.go diff --git a/pkg/omni/full/retriever/block_retriever_test.go b/pkg/contract_watcher/full/retriever/block_retriever_test.go similarity index 97% rename from pkg/omni/full/retriever/block_retriever_test.go rename to pkg/contract_watcher/full/retriever/block_retriever_test.go index dcd13c35..4cc4c04e 100644 --- a/pkg/omni/full/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/full/retriever/block_retriever_test.go @@ -21,12 +21,12 @@ import ( . "github.com/onsi/gomega" "strings" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) var _ = Describe("Block Retriever", func() { diff --git a/pkg/omni/full/retriever/retriever_suite_test.go b/pkg/contract_watcher/full/retriever/retriever_suite_test.go similarity index 100% rename from pkg/omni/full/retriever/retriever_suite_test.go rename to pkg/contract_watcher/full/retriever/retriever_suite_test.go diff --git a/pkg/contract_watcher/full/transformer/transformer.go b/pkg/contract_watcher/full/transformer/transformer.go new file mode 100644 index 00000000..ad2bba69 --- /dev/null +++ b/pkg/contract_watcher/full/transformer/transformer.go @@ -0,0 +1,226 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer + +import ( + "errors" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" +) + +// Requires a fully synced vDB and a running eth node (or infura) +type Transformer struct { + // Database interfaces + FilterRepository datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() + WatchedEventRepository datastore.WatchedEventRepository // Watched event log views, created by the log filters + TransformedEventRepository repository.EventRepository // Holds transformed watched event log data + + // Pre-processing interfaces + Parser parser.Parser // Parses events and methods out of contract abi fetched using contract address + Retriever retriever.BlockRetriever // Retrieves first block for contract and current block height + + // Processing interfaces + Converter converter.ConverterInterface // Converts watched event logs into custom log + Poller poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore + + // Store contract configuration information + Config config.ContractConfig + + // Store contract info as mapping to contract address + Contracts map[string]*contract.Contract + + // Latest block in the block repository + LastBlock int64 +} + +// Transformer takes in config for blockchain, database, and network id +func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer { + return &Transformer{ + Poller: poller.NewPoller(BC, DB, types.FullSync), + Parser: parser.NewParser(con.Network), + Retriever: retriever.NewBlockRetriever(DB), + Converter: &converter.Converter{}, + Contracts: map[string]*contract.Contract{}, + WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, + FilterRepository: repositories.FilterRepository{DB: DB}, + TransformedEventRepository: repository.NewEventRepository(DB, types.FullSync), + Config: con, + } +} + +// Use after creating and setting transformer +// Loops over all of the addr => filter sets +// Uses parser to pull event info from abi +// Use this info to generate event filters +func (tr *Transformer) Init() error { + for contractAddr := range tr.Config.Addresses { + // Configure Abi + if tr.Config.Abis[contractAddr] == "" { + // If no abi is given in the config, this method will try fetching from internal look-up table and etherscan + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + } else { + // If we have an abi from the config, load that into the parser + err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr]) + if err != nil { + return err + } + } + + // Get first block and most recent block number in the header repo + firstBlock, err := tr.Retriever.RetrieveFirstBlock(contractAddr) + if err != nil { + return err + } + // Set to specified range if it falls within the bounds + if firstBlock < tr.Config.StartingBlocks[contractAddr] { + firstBlock = tr.Config.StartingBlocks[contractAddr] + } + + // Get contract name if it has one + var name = new(string) + tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock) + + // Remove any potential accidental duplicate inputs in arg filter values + eventArgs := map[string]bool{} + for _, arg := range tr.Config.EventArgs[contractAddr] { + eventArgs[arg] = true + } + methodArgs := map[string]bool{} + for _, arg := range tr.Config.MethodArgs[contractAddr] { + methodArgs[arg] = true + } + + // Aggregate info into contract object + info := contract.Contract{ + Name: *name, + Network: tr.Config.Network, + Address: contractAddr, + Abi: tr.Parser.Abi(), + ParsedAbi: tr.Parser.ParsedAbi(), + StartingBlock: firstBlock, + Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), + Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + Piping: tr.Config.Piping[contractAddr], + }.Init() + + // Use info to create filters + err = info.GenerateFilters() + if err != nil { + return err + } + + // Iterate over filters and push them to the repo using filter repository interface + for _, filter := range info.Filters { + err = tr.FilterRepository.CreateFilter(filter) + if err != nil { + return err + } + } + + // Store contract info for further processing + tr.Contracts[contractAddr] = info + } + + // Get the most recent block number in the block repo + var err error + tr.LastBlock, err = tr.Retriever.RetrieveMostRecentBlock() + if err != nil { + return err + } + + return nil +} + +// Iterates through stored, initialized contract objects +// Iterates through contract's event filters, grabbing watched event logs +// Uses converter to convert logs into custom log type +// Persists converted logs into custuom postgres tables +// Calls selected methods, using token holder address generated during event log conversion +func (tr *Transformer) Execute() error { + if len(tr.Contracts) == 0 { + return errors.New("error: transformer has no initialized contracts to work with") + } + // Iterate through all internal contracts + for _, con := range tr.Contracts { + // Update converter with current contract + tr.Converter.Update(con) + + // Iterate through contract filters and get watched event logs + for eventSig, filter := range con.Filters { + watchedEvents, err := tr.WatchedEventRepository.GetWatchedEvents(filter.Name) + if err != nil { + return err + } + + // Iterate over watched event logs + for _, we := range watchedEvents { + // Convert them to our custom log type + cstm, err := tr.Converter.Convert(*we, con.Events[eventSig]) + if err != nil { + return err + } + if cstm == nil { + continue + } + + // If log is not empty, immediately persist in repo + // Run this in seperate goroutine? + err = tr.TransformedEventRepository.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) + if err != nil { + return err + } + } + } + + // After persisting all watched event logs + // poller polls select contract methods + // and persists the results into custom pg tables + if err := tr.Poller.PollContract(*con, tr.LastBlock); err != nil { + return err + } + } + + // At the end of a transformation cycle, and before the next + // update the latest block from the block repo + var err error + tr.LastBlock, err = tr.Retriever.RetrieveMostRecentBlock() + if err != nil { + return err + } + + return nil +} + +func (tr *Transformer) GetConfig() config.ContractConfig { + return tr.Config +} diff --git a/pkg/omni/full/transformer/transformer_suite_test.go b/pkg/contract_watcher/full/transformer/transformer_suite_test.go similarity index 100% rename from pkg/omni/full/transformer/transformer_suite_test.go rename to pkg/contract_watcher/full/transformer/transformer_suite_test.go diff --git a/pkg/contract_watcher/full/transformer/transformer_test.go b/pkg/contract_watcher/full/transformer/transformer_test.go new file mode 100644 index 00000000..ac03e1a6 --- /dev/null +++ b/pkg/contract_watcher/full/transformer/transformer_test.go @@ -0,0 +1,98 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer_test + +import ( + "math/rand" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" +) + +var _ = Describe("Transformer", func() { + var fakeAddress = "0x1234567890abcdef" + rand.Seed(time.Now().UnixNano()) + + Describe("Init", func() { + It("Initializes transformer's contract objects", func() { + blockRetriever := &fakes.MockFullBlockRetriever{} + firstBlock := int64(1) + mostRecentBlock := int64(2) + blockRetriever.FirstBlock = firstBlock + blockRetriever.MostRecentBlock = mostRecentBlock + + parsr := &fakes.MockParser{} + fakeAbi := "fake_abi" + eventName := "Transfer" + event := types.Event{} + parsr.AbiToReturn = fakeAbi + parsr.EventName = eventName + parsr.Event = event + + pollr := &fakes.MockPoller{} + fakeContractName := "fake_contract_name" + pollr.ContractName = fakeContractName + + t := getTransformer(blockRetriever, parsr, pollr) + + err := t.Init() + + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[fakeAddress] + Expect(ok).To(Equal(true)) + + Expect(c.StartingBlock).To(Equal(firstBlock)) + Expect(t.LastBlock).To(Equal(mostRecentBlock)) + Expect(c.Abi).To(Equal(fakeAbi)) + Expect(c.Name).To(Equal(fakeContractName)) + Expect(c.Address).To(Equal(fakeAddress)) + }) + + It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { + blockRetriever := &fakes.MockFullBlockRetriever{} + blockRetriever.FirstBlockErr = fakes.FakeError + t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) + + err := t.Init() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) +}) + +func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { + return transformer.Transformer{ + FilterRepository: &fakes.MockFilterRepository{}, + Parser: parsr, + Retriever: blockRetriever, + Poller: pollr, + Contracts: map[string]*contract.Contract{}, + Config: mocks.MockConfig, + } +} diff --git a/pkg/omni/light/converter/converter.go b/pkg/contract_watcher/light/converter/converter.go similarity index 90% rename from pkg/omni/light/converter/converter.go rename to pkg/contract_watcher/light/converter/converter.go index b2c0d7ac..3a62c807 100644 --- a/pkg/omni/light/converter/converter.go +++ b/pkg/contract_watcher/light/converter/converter.go @@ -28,32 +28,26 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" gethTypes "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) -type Converter interface { +type ConverterInterface interface { Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) Update(info *contract.Contract) } -type converter struct { +type Converter struct { ContractInfo *contract.Contract } -func NewConverter(info *contract.Contract) *converter { - return &converter{ - ContractInfo: info, - } -} - -func (c *converter) Update(info *contract.Contract) { +func (c *Converter) Update(info *contract.Contract) { c.ContractInfo = info } // Convert the given watched event log into a types.Log for the given event -func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { +func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) returnLogs := make([]types.Log, 0, len(logs)) for _, log := range logs { @@ -132,8 +126,8 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in } // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs -func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) +func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) eventsToLogs := make(map[string][]types.Log) for _, event := range events { eventsToLogs[event.Name] = make([]types.Log, 0, len(logs)) @@ -142,7 +136,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E // If the log is of this event type, process it as such if event.Sig() == log.Topics[0] { values := make(map[string]interface{}) - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/omni/light/converter/converter_suite_test.go b/pkg/contract_watcher/light/converter/converter_suite_test.go similarity index 100% rename from pkg/omni/light/converter/converter_suite_test.go rename to pkg/contract_watcher/light/converter/converter_suite_test.go diff --git a/pkg/omni/light/converter/converter_test.go b/pkg/contract_watcher/light/converter/converter_test.go similarity index 89% rename from pkg/omni/light/converter/converter_test.go rename to pkg/contract_watcher/light/converter/converter_test.go index 67c9950d..769fe057 100644 --- a/pkg/omni/light/converter/converter_test.go +++ b/pkg/contract_watcher/light/converter/converter_test.go @@ -22,11 +22,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" ) var _ = Describe("Converter", func() { @@ -38,7 +38,8 @@ var _ = Describe("Converter", func() { Describe("Update", func() { It("Updates contract info held by the converter", func() { con = test_helpers.SetupTusdContract(tusdWantedEvents, []string{}) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) Expect(c.ContractInfo).To(Equal(con)) info := test_helpers.SetupTusdContract([]string{}, []string{}) @@ -56,7 +57,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["Transfer"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) logs, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) @@ -80,7 +82,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["Transfer"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) _, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) @@ -110,7 +113,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["NewOwner"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) _, err := c.Convert([]types.Log{mocks.MockNewOwnerLog1, mocks.MockNewOwnerLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) Expect(len(con.EmittedHashes)).To(Equal(3)) @@ -143,7 +147,8 @@ var _ = Describe("Converter", func() { It("Fails with an empty contract", func() { event := con.Events["Transfer"] - c := converter.NewConverter(&contract.Contract{}) + c := converter.Converter{} + c.Update(&contract.Contract{}) _, err = c.Convert([]types.Log{mocks.MockTransferLog1}, event, 232) Expect(err).To(HaveOccurred()) }) diff --git a/pkg/omni/light/fetcher/fetcher.go b/pkg/contract_watcher/light/fetcher/fetcher.go similarity index 100% rename from pkg/omni/light/fetcher/fetcher.go rename to pkg/contract_watcher/light/fetcher/fetcher.go diff --git a/pkg/omni/light/fetcher/fetcher_suite_test.go b/pkg/contract_watcher/light/fetcher/fetcher_suite_test.go similarity index 100% rename from pkg/omni/light/fetcher/fetcher_suite_test.go rename to pkg/contract_watcher/light/fetcher/fetcher_suite_test.go diff --git a/pkg/omni/light/fetcher/fetcher_test.go b/pkg/contract_watcher/light/fetcher/fetcher_test.go similarity index 96% rename from pkg/omni/light/fetcher/fetcher_test.go rename to pkg/contract_watcher/light/fetcher/fetcher_test.go index 77982d84..b3b9b96c 100644 --- a/pkg/omni/light/fetcher/fetcher_test.go +++ b/pkg/contract_watcher/light/fetcher/fetcher_test.go @@ -22,9 +22,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/fetcher" ) var _ = Describe("Fetcher", func() { diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/contract_watcher/light/repository/header_repository.go similarity index 82% rename from pkg/omni/light/repository/header_repository.go rename to pkg/contract_watcher/light/repository/header_repository.go index 87d81990..3af7652c 100644 --- a/pkg/omni/light/repository/header_repository.go +++ b/pkg/contract_watcher/light/repository/header_repository.go @@ -53,6 +53,7 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository { } } +// Adds a checked_header column for the provided column id func (r *headerRepository) AddCheckColumn(id string) error { // Check cache to see if column already exists before querying pg _, ok := r.columns.Get(id) @@ -73,6 +74,7 @@ func (r *headerRepository) AddCheckColumn(id string) error { return nil } +// Adds a checked_header column for all of the provided column ids func (r *headerRepository) AddCheckColumns(ids []string) error { var err error baseQuery := "ALTER TABLE public.checked_headers" @@ -96,6 +98,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error { return err } +// Marks the header checked for the provided column id func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`) VALUES ($1, $2) @@ -105,6 +108,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { return err } +// Marks the header checked for all of the provided column ids func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error { pgStr := "INSERT INTO public.checked_headers (header_id, " for _, id := range ids { @@ -124,6 +128,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) return err } +// Marks all of the provided headers checked for each of the provided column ids func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { tx, err := r.db.Beginx() if err != nil { @@ -154,6 +159,7 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [ return tx.Commit() } +// Returns missing headers for the provided checked_headers column id func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) { var result []core.Header var query string @@ -165,7 +171,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber WHERE (header_id ISNULL OR checked_headers.` + id + `=0) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers @@ -174,13 +180,14 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } - return result, err + return contiguousHeaders(result, startingBlockNumber), err } +// Returns missing headers for all of the provided checked_headers column ids func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) { var result []core.Header var query string @@ -196,21 +203,42 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock if endingBlockNumber == -1 { endStr := `) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { endStr := `) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } - return result, err + return contiguousHeaders(result, startingBlockNumber), err } +// Takes in an ordered sequence of headers and returns only the first contiguous segment +// Enforce continuity with previous segment with the appropriate startingBlockNumber +func contiguousHeaders(headers []core.Header, startingBlockNumber int64) []core.Header { + if len(headers) < 1 { + return headers + } + previousHeader := headers[0].BlockNumber + if previousHeader != startingBlockNumber { + return []core.Header{} + } + for i := 1; i < len(headers); i++ { + previousHeader++ + if headers[i].BlockNumber != previousHeader { + return headers[:i] + } + } + + return headers +} + +// Returns headers that have been checked for all of the provided event ids but not for the provided method ids func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) { var result []core.Header var query string @@ -231,14 +259,14 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock if endingBlockNumber == -1 { endStr := `AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { endStr := `AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } @@ -246,10 +274,12 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock return result, err } +// Check the repositories column id cache for a value func (r *headerRepository) CheckCache(key string) (interface{}, bool) { return r.columns.Get(key) } +// Used to mark a header checked as part of some external transaction so as to group into one commit func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) VALUES ($1, $2) diff --git a/pkg/omni/light/repository/header_repository_test.go b/pkg/contract_watcher/light/repository/header_repository_test.go similarity index 56% rename from pkg/omni/light/repository/header_repository_test.go rename to pkg/contract_watcher/light/repository/header_repository_test.go index 6c4958b7..01eb876a 100644 --- a/pkg/omni/light/repository/header_repository_test.go +++ b/pkg/contract_watcher/light/repository/header_repository_test.go @@ -23,17 +23,17 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Repository", func() { var db *postgres.DB - var omniHeaderRepo repository.HeaderRepository // omni/light header repository - var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository + var contractHeaderRepo repository.HeaderRepository // contract_watcher light header repository + var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository var eventIDs = []string{ "eventName_contractAddr", "eventName_contractAddr2", @@ -47,7 +47,7 @@ var _ = Describe("Repository", func() { BeforeEach(func() { db, _ = test_helpers.SetupDBandBC() - omniHeaderRepo = repository.NewHeaderRepository(db) + contractHeaderRepo = repository.NewHeaderRepository(db) coreHeaderRepo = repositories.NewHeaderRepository(db) }) @@ -61,7 +61,7 @@ var _ = Describe("Repository", func() { _, err := db.Exec(query) Expect(err).To(HaveOccurred()) - err = omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err = contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) _, err = db.Exec(query) @@ -69,13 +69,13 @@ var _ = Describe("Repository", func() { }) It("Caches column it creates so that it does not need to repeatedly query the database to check for it's existence", func() { - _, ok := omniHeaderRepo.CheckCache(eventIDs[0]) + _, ok := contractHeaderRepo.CheckCache(eventIDs[0]) Expect(ok).To(Equal(false)) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - v, ok := omniHeaderRepo.CheckCache(eventIDs[0]) + v, ok := contractHeaderRepo.CheckCache(eventIDs[0]) Expect(ok).To(Equal(true)) Expect(v).To(Equal(true)) }) @@ -88,7 +88,7 @@ var _ = Describe("Repository", func() { Expect(err).To(HaveOccurred()) } - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range eventIDs { @@ -99,15 +99,15 @@ var _ = Describe("Repository", func() { It("Caches columns it creates so that it does not need to repeatedly query the database to check for it's existence", func() { for _, id := range eventIDs { - _, ok := omniHeaderRepo.CheckCache(id) + _, ok := contractHeaderRepo.CheckCache(id) Expect(ok).To(Equal(false)) } - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range eventIDs { - v, ok := omniHeaderRepo.CheckCache(id) + v, ok := contractHeaderRepo.CheckCache(id) Expect(ok).To(Equal(true)) Expect(v).To(Equal(true)) } @@ -117,20 +117,20 @@ var _ = Describe("Repository", func() { Describe("MissingHeaders", func() { It("Returns all unchecked headers for the given eventID", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) }) It("Returns unchecked headers in ascending order", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -142,12 +142,24 @@ var _ = Describe("Repository", func() { Expect(h3.BlockNumber).To(Equal(int64(6194634))) }) - It("Fails if eventID does not yet exist in check_headers table", func() { - addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + It("Returns only contiguous chunks of headers", func() { + addDiscontinuousHeaders(coreHeaderRepo) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - _, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, "notEventId") + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(2)) + Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) + Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633))) + }) + + It("Fails if eventID does not yet exist in check_headers table", func() { + addHeaders(coreHeaderRepo) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) + Expect(err).ToNot(HaveOccurred()) + + _, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, "notEventId") Expect(err).To(HaveOccurred()) }) }) @@ -155,37 +167,61 @@ var _ = Describe("Repository", func() { Describe("MissingHeadersForAll", func() { // HERE It("Returns all headers that have not been checked for all of the ids provided", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) - err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0]) + err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) - err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[1]) + err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[1]) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2]) + err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194633, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) + It("Returns only contiguous chunks of headers", func() { + addDiscontinuousHeaders(coreHeaderRepo) + err := contractHeaderRepo.AddCheckColumns(eventIDs) + Expect(err).ToNot(HaveOccurred()) + + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(2)) + Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) + Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633))) + }) + + It("Returns at most 100 headers", func() { + add102Headers(coreHeaderRepo) + err := contractHeaderRepo.AddCheckColumns(eventIDs) + Expect(err).ToNot(HaveOccurred()) + + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194733, eventIDs) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(100)) + Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) + Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633))) + }) + It("Fails if one of the eventIDs does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) badEventIDs := append(eventIDs, "notEventId") - _, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, badEventIDs) + _, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, badEventIDs) Expect(err).To(HaveOccurred()) }) }) @@ -193,36 +229,36 @@ var _ = Describe("Repository", func() { Describe("MarkHeaderChecked", func() { It("Marks the header checked for the given eventID", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id - err = omniHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0]) + err = contractHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) It("Fails if eventID does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id - err = omniHeaderRepo.MarkHeaderChecked(headerID, "notEventId") + err = contractHeaderRepo.MarkHeaderChecked(headerID, "notEventId") Expect(err).To(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) }) @@ -231,18 +267,18 @@ var _ = Describe("Repository", func() { Describe("MarkHeaderCheckedForAll", func() { It("Marks the header checked for all provided column ids", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id - err = omniHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs) + err = contractHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) @@ -259,17 +295,17 @@ var _ = Describe("Repository", func() { var missingHeaders []core.Header for _, id := range methodIDs { - err := omniHeaderRepo.AddCheckColumn(id) + err := contractHeaderRepo.AddCheckColumn(id) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) } - err := omniHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs) + err := contractHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range methodIDs { - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(0)) } @@ -280,28 +316,28 @@ var _ = Describe("Repository", func() { It("Returns headers that have been checked for all the provided events but have not been checked for all the provided methods", func() { addHeaders(coreHeaderRepo) for i, id := range eventIDs { - err := omniHeaderRepo.AddCheckColumn(id) + err := contractHeaderRepo.AddCheckColumn(id) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.AddCheckColumn(methodIDs[i]) + err = contractHeaderRepo.AddCheckColumn(methodIDs[i]) Expect(err).ToNot(HaveOccurred()) } - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id headerID2 := missingHeaders[1].Id for i, id := range eventIDs { - err = omniHeaderRepo.MarkHeaderChecked(headerID, id) + err = contractHeaderRepo.MarkHeaderChecked(headerID, id) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.MarkHeaderChecked(headerID2, id) + err = contractHeaderRepo.MarkHeaderChecked(headerID2, id) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.MarkHeaderChecked(headerID, methodIDs[i]) + err = contractHeaderRepo.MarkHeaderChecked(headerID, methodIDs[i]) Expect(err).ToNot(HaveOccurred()) } - intersectionHeaders, err := omniHeaderRepo.MissingMethodsCheckedEventsIntersection(6194630, 6194635, methodIDs, eventIDs) + intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(6194632, 6194635, methodIDs, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(intersectionHeaders)).To(Equal(1)) Expect(intersectionHeaders[0].Id).To(Equal(headerID2)) @@ -315,3 +351,18 @@ func addHeaders(coreHeaderRepo repositories.HeaderRepository) { coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2) coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3) } + +func addDiscontinuousHeaders(coreHeaderRepo repositories.HeaderRepository) { + coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1) + coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2) + coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4) +} + +func add102Headers(coreHeaderRepo repositories.HeaderRepository) { + baseHeader := mocks.MockHeader1 + for i := 6194632; i < 6194733; i++ { + _, err := coreHeaderRepo.CreateOrUpdateHeader(baseHeader) + Expect(err).ToNot(HaveOccurred()) + baseHeader.BlockNumber++ + } +} diff --git a/pkg/omni/light/repository/repository_suite_test.go b/pkg/contract_watcher/light/repository/repository_suite_test.go similarity index 100% rename from pkg/omni/light/repository/repository_suite_test.go rename to pkg/contract_watcher/light/repository/repository_suite_test.go diff --git a/pkg/omni/light/retriever/block_retriever.go b/pkg/contract_watcher/light/retriever/block_retriever.go similarity index 100% rename from pkg/omni/light/retriever/block_retriever.go rename to pkg/contract_watcher/light/retriever/block_retriever.go diff --git a/pkg/omni/light/retriever/block_retriever_test.go b/pkg/contract_watcher/light/retriever/block_retriever_test.go similarity index 90% rename from pkg/omni/light/retriever/block_retriever_test.go rename to pkg/contract_watcher/light/retriever/block_retriever_test.go index d9066b92..a57852e5 100644 --- a/pkg/omni/light/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/light/retriever/block_retriever_test.go @@ -20,11 +20,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Block Retriever", func() { diff --git a/pkg/omni/light/retriever/retriever_suite_test.go b/pkg/contract_watcher/light/retriever/retriever_suite_test.go similarity index 100% rename from pkg/omni/light/retriever/retriever_suite_test.go rename to pkg/contract_watcher/light/retriever/retriever_suite_test.go diff --git a/pkg/omni/light/transformer/transformer.go b/pkg/contract_watcher/light/transformer/transformer.go similarity index 58% rename from pkg/omni/light/transformer/transformer.go rename to pkg/contract_watcher/light/transformer/transformer.go index f7738c0a..8ce5f064 100644 --- a/pkg/omni/light/transformer/transformer.go +++ b/pkg/contract_watcher/light/transformer/transformer.go @@ -23,67 +23,48 @@ import ( "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + srep "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/fetcher" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - srep "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Requires a light synced vDB (headers) and a running eth node (or infura) type Transformer struct { // Database interfaces - srep.EventRepository // Holds transformed watched event log data - repository.HeaderRepository // Interface for interaction with header repositories + EventRepository srep.EventRepository // Holds transformed watched event log data + HeaderRepository repository.HeaderRepository // Interface for interaction with header repositories // Pre-processing interfaces - parser.Parser // Parses events and methods out of contract abi fetched using contract address - retriever.BlockRetriever // Retrieves first block for contract and current block height + Parser parser.Parser // Parses events and methods out of contract abi fetched using contract address + Retriever retriever.BlockRetriever // Retrieves first block for contract // Processing interfaces - fetcher.Fetcher // Fetches event logs, using header hashes - converter.Converter // Converts watched event logs into custom log - poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore + Fetcher fetcher.Fetcher // Fetches event logs, using header hashes + Converter converter.ConverterInterface // Converts watched event logs into custom log + Poller poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore - // Ethereum network name; default "" is mainnet - Network string + // Store contract configuration information + Config config.ContractConfig // Store contract info as mapping to contract address Contracts map[string]*contract.Contract - // Targeted subset of events/methods - // Stored as maps of contract address to events/method names of interest - WatchedEvents map[string][]string // Default/empty event list means all are watched - WantedMethods map[string][]string // Default/empty method list means none are polled - - // Starting block number for each contract - ContractStart map[string]int64 - - // Lists of argument values to filter event or - // method data with; if empty no filter is applied - EventArgs map[string][]string - MethodArgs map[string][]string - - // Whether or not to create a list of emitted address or hashes for the contract in postgres - CreateAddrList map[string]bool - CreateHashList map[string]bool - - // Method piping on/off for a contract - Piping map[string]bool - // Internally configured transformer variables contractAddresses []string // Holds all contract addresses, for batch fetching of logs sortedEventIds map[string][]string // Map to sort event column ids by contract, for post fetch processing and persisting of logs sortedMethodIds map[string][]string // Map to sort method column ids by contract, for post fetch method polling eventIds []string // Holds event column ids across all contract, for batch fetching of headers eventFilters []common.Hash // Holds topic0 hashes across all contracts, for batch fetching of logs - start int64 // Hold the lowest starting block and the highest ending block + Start int64 // Hold the lowest starting block and the highest ending block } // Order-of-operations: @@ -93,26 +74,18 @@ type Transformer struct { // 4. Execute // Transformer takes in config for blockchain, database, and network id -func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer { +func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer { return &Transformer{ Poller: poller.NewPoller(bc, db, types.LightSync), Fetcher: fetcher.NewFetcher(bc), - Parser: parser.NewParser(network), + Parser: parser.NewParser(con.Network), HeaderRepository: repository.NewHeaderRepository(db), - BlockRetriever: retriever.NewBlockRetriever(db), - Converter: converter.NewConverter(&contract.Contract{}), + Retriever: retriever.NewBlockRetriever(db), + Converter: &converter.Converter{}, Contracts: map[string]*contract.Contract{}, EventRepository: srep.NewEventRepository(db, types.LightSync), - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - Piping: map[string]bool{}, - Network: network, + Config: con, } } @@ -127,61 +100,63 @@ func (tr *Transformer) Init() error { tr.sortedMethodIds = make(map[string][]string) // Map to sort method column ids by contract, for post fetch method polling tr.eventIds = make([]string, 0) // Holds event column ids across all contract, for batch fetching of headers tr.eventFilters = make([]common.Hash, 0) // Holds topic0 hashes across all contracts, for batch fetching of logs - tr.start = 100000000000 // Hold the lowest starting block and the highest ending block + tr.Start = 100000000000 // Iterate through all internal contract addresses - for contractAddr, subset := range tr.WatchedEvents { - // Get Abi - err := tr.Parser.Parse(contractAddr) - if err != nil { - return err + for contractAddr := range tr.Config.Addresses { + // Configure Abi + if tr.Config.Abis[contractAddr] == "" { + // If no abi is given in the config, this method will try fetching from internal look-up table and etherscan + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + } else { + // If we have an abi from the config, load that into the parser + err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr]) + if err != nil { + return err + } } // Get first block and most recent block number in the header repo - firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock() - if err != nil { - return err - } - lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock() + firstBlock, err := tr.Retriever.RetrieveFirstBlock() if err != nil { return err } // Set to specified range if it falls within the bounds - if firstBlock < tr.ContractStart[contractAddr] { - firstBlock = tr.ContractStart[contractAddr] + if firstBlock < tr.Config.StartingBlocks[contractAddr] { + firstBlock = tr.Config.StartingBlocks[contractAddr] } // Get contract name if it has one var name = new(string) - tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock) + tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1) - // Remove any potential accidental duplicate inputs in arg filter values + // Remove any potential accidental duplicate inputs eventArgs := map[string]bool{} - for _, arg := range tr.EventArgs[contractAddr] { + for _, arg := range tr.Config.EventArgs[contractAddr] { eventArgs[arg] = true } methodArgs := map[string]bool{} - for _, arg := range tr.MethodArgs[contractAddr] { + for _, arg := range tr.Config.MethodArgs[contractAddr] { methodArgs[arg] = true } // Aggregate info into contract object and store for execution con := contract.Contract{ - Name: *name, - Network: tr.Network, - Address: contractAddr, - Abi: tr.Parser.Abi(), - ParsedAbi: tr.Parser.ParsedAbi(), - StartingBlock: firstBlock, - LastBlock: -1, - Events: tr.Parser.GetEvents(subset), - Methods: tr.Parser.GetSelectMethods(tr.WantedMethods[contractAddr]), - FilterArgs: eventArgs, - MethodArgs: methodArgs, - CreateAddrList: tr.CreateAddrList[contractAddr], - CreateHashList: tr.CreateHashList[contractAddr], - Piping: tr.Piping[contractAddr], + Name: *name, + Network: tr.Config.Network, + Address: contractAddr, + Abi: tr.Parser.Abi(), + ParsedAbi: tr.Parser.ParsedAbi(), + StartingBlock: firstBlock, + Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), + Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + Piping: tr.Config.Piping[contractAddr], }.Init() tr.Contracts[contractAddr] = con tr.contractAddresses = append(tr.contractAddresses, con.Address) @@ -213,8 +188,8 @@ func (tr *Transformer) Init() error { } // Update start to the lowest block - if con.StartingBlock < tr.start { - tr.start = con.StartingBlock + if con.StartingBlock < tr.Start { + tr.Start = con.StartingBlock } } @@ -226,20 +201,20 @@ func (tr *Transformer) Execute() error { return errors.New("error: transformer has no initialized contracts") } - // Map to sort batch fetched logs by which contract they belong to, for post fetch processing - sortedLogs := make(map[string][]gethTypes.Log) - for _, con := range tr.Contracts { - sortedLogs[con.Address] = []gethTypes.Log{} - } - // Find unchecked headers for all events across all contracts; these are returned in asc order - missingHeaders, err := tr.HeaderRepository.MissingHeadersForAll(tr.start, -1, tr.eventIds) + missingHeaders, err := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds) if err != nil { return err } // Iterate over headers for _, header := range missingHeaders { + // Set `start` to this header + // This way if we throw an error but don't bring the execution cycle down (how it is currently handled) + // we restart the cycle at this header + tr.Start = header.BlockNumber + // Map to sort batch fetched logs by which contract they belong to, for post fetch processing + sortedLogs := make(map[string][]gethTypes.Log) // And fetch all event logs across contracts at this header allLogs, err := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header) if err != nil { @@ -257,6 +232,7 @@ func (tr *Transformer) Execute() error { if err != nil { return err } + tr.Start = header.BlockNumber + 1 // Empty header; setup to start at the next header continue } @@ -305,6 +281,8 @@ func (tr *Transformer) Execute() error { if err != nil { return err } + // Success; setup to start at the next header + tr.Start = header.BlockNumber + 1 } return nil @@ -335,42 +313,6 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str return nil } -// Used to set which contract addresses and which of their events to watch -func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) { - tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to watch events for -func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) { - tr.EventArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set which contract addresses and which of their methods to call -func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) { - tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to poll methods on -func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) { - tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set the block range to watch for a given address -func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) { - tr.ContractStart[strings.ToLower(contractAddr)] = start -} - -// Used to set whether or not to persist an account address list -func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) { - tr.CreateAddrList[strings.ToLower(contractAddr)] = on -} - -// Used to set whether or not to persist an hash list -func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) { - tr.CreateHashList[strings.ToLower(contractAddr)] = on -} - -// Used to turn method piping on for a contract -func (tr *Transformer) SetPiping(contractAddr string, on bool) { - tr.Piping[strings.ToLower(contractAddr)] = on +func (tr *Transformer) GetConfig() config.ContractConfig { + return tr.Config } diff --git a/pkg/omni/light/transformer/transformer_suite_test.go b/pkg/contract_watcher/light/transformer/transformer_suite_test.go similarity index 100% rename from pkg/omni/light/transformer/transformer_suite_test.go rename to pkg/contract_watcher/light/transformer/transformer_suite_test.go diff --git a/pkg/contract_watcher/light/transformer/transformer_test.go b/pkg/contract_watcher/light/transformer/transformer_test.go new file mode 100644 index 00000000..8c17f570 --- /dev/null +++ b/pkg/contract_watcher/light/transformer/transformer_test.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/fakes" +) + +var _ = Describe("Transformer", func() { + var fakeAddress = "0x1234567890abcdef" + Describe("Init", func() { + It("Initializes transformer's contract objects", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + firstBlock := int64(1) + blockRetriever.FirstBlock = firstBlock + + parsr := &fakes.MockParser{} + fakeAbi := "fake_abi" + parsr.AbiToReturn = fakeAbi + + pollr := &fakes.MockPoller{} + fakeContractName := "fake_contract_name" + pollr.ContractName = fakeContractName + + t := getFakeTransformer(blockRetriever, parsr, pollr) + + err := t.Init() + + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[fakeAddress] + Expect(ok).To(Equal(true)) + + Expect(c.StartingBlock).To(Equal(firstBlock)) + Expect(c.Abi).To(Equal(fakeAbi)) + Expect(c.Name).To(Equal(fakeContractName)) + Expect(c.Address).To(Equal(fakeAddress)) + }) + + It("Fails to initialize if first block cannot be fetched from vDB headers table", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + blockRetriever.FirstBlockErr = fakes.FakeError + t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) + + err := t.Init() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) + + Describe("Execute", func() { + It("Executes contract transformations", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + firstBlock := int64(1) + blockRetriever.FirstBlock = firstBlock + + parsr := &fakes.MockParser{} + fakeAbi := "fake_abi" + parsr.AbiToReturn = fakeAbi + + pollr := &fakes.MockPoller{} + fakeContractName := "fake_contract_name" + pollr.ContractName = fakeContractName + + t := getFakeTransformer(blockRetriever, parsr, pollr) + + err := t.Init() + + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[fakeAddress] + Expect(ok).To(Equal(true)) + + Expect(c.StartingBlock).To(Equal(firstBlock)) + Expect(c.Abi).To(Equal(fakeAbi)) + Expect(c.Name).To(Equal(fakeContractName)) + Expect(c.Address).To(Equal(fakeAddress)) + }) + + It("Fails to initialize if first block cannot be fetched from vDB headers table", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + blockRetriever.FirstBlockErr = fakes.FakeError + t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) + + err := t.Init() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) +}) + +func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { + return transformer.Transformer{ + Parser: parsr, + Retriever: blockRetriever, + Poller: pollr, + HeaderRepository: &fakes.MockLightHeaderRepository{}, + Contracts: map[string]*contract.Contract{}, + Config: mocks.MockConfig, + } +} diff --git a/pkg/omni/shared/constants/constants.go b/pkg/contract_watcher/shared/constants/constants.go similarity index 99% rename from pkg/omni/shared/constants/constants.go rename to pkg/contract_watcher/shared/constants/constants.go index 9ac4bd80..5545f791 100644 --- a/pkg/omni/shared/constants/constants.go +++ b/pkg/contract_watcher/shared/constants/constants.go @@ -18,9 +18,9 @@ package constants import ( "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) // Event enums @@ -72,7 +72,6 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E" var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b" var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5" -// TODO: Consider whether these should be moved to plugins // Contract Owner var DaiContractOwner = "0x0000000000000000000000000000000000000000" var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f" diff --git a/pkg/omni/shared/constants/interface.go b/pkg/contract_watcher/shared/constants/interface.go similarity index 100% rename from pkg/omni/shared/constants/interface.go rename to pkg/contract_watcher/shared/constants/interface.go diff --git a/pkg/omni/shared/contract/contract.go b/pkg/contract_watcher/shared/contract/contract.go similarity index 93% rename from pkg/omni/shared/contract/contract.go rename to pkg/contract_watcher/shared/contract/contract.go index 19364d91..6b3eb0ff 100644 --- a/pkg/omni/shared/contract/contract.go +++ b/pkg/contract_watcher/shared/contract/contract.go @@ -23,9 +23,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Contract object to hold our contract data @@ -34,7 +34,6 @@ type Contract struct { Address string // Address of the contract Network string // Network on which the contract is deployed; default empty "" is Ethereum mainnet StartingBlock int64 // Starting block of the contract - LastBlock int64 // Most recent block on the network Abi string // Abi string ParsedAbi abi.ABI // Parsed abi Events map[string]types.Event // List of events to watch @@ -64,16 +63,10 @@ func (c Contract) Init() *Contract { } } - // If we are creating an address list in postgres - // we initialize the map despite what method call, if any - if c.CreateAddrList { - c.EmittedAddrs = map[interface{}]bool{} - } - return &c } -// Use contract info to generate event filters - full sync omni watcher only +// Use contract info to generate event filters - full sync contract watcher only func (c *Contract) GenerateFilters() error { c.Filters = map[string]filters.LogFilter{} diff --git a/pkg/omni/shared/contract/contract_suite_test.go b/pkg/contract_watcher/shared/contract/contract_suite_test.go similarity index 100% rename from pkg/omni/shared/contract/contract_suite_test.go rename to pkg/contract_watcher/shared/contract/contract_suite_test.go diff --git a/pkg/omni/shared/contract/contract_test.go b/pkg/contract_watcher/shared/contract/contract_test.go similarity index 95% rename from pkg/omni/shared/contract/contract_test.go rename to pkg/contract_watcher/shared/contract/contract_test.go index 819e6959..38a996ec 100644 --- a/pkg/omni/shared/contract/contract_test.go +++ b/pkg/contract_watcher/shared/contract/contract_test.go @@ -20,10 +20,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) var _ = Describe("Contract", func() { diff --git a/pkg/omni/shared/fetcher/fetcher.go b/pkg/contract_watcher/shared/fetcher/fetcher.go similarity index 100% rename from pkg/omni/shared/fetcher/fetcher.go rename to pkg/contract_watcher/shared/fetcher/fetcher.go diff --git a/pkg/omni/shared/getter/getter_suite_test.go b/pkg/contract_watcher/shared/getter/getter_suite_test.go similarity index 100% rename from pkg/omni/shared/getter/getter_suite_test.go rename to pkg/contract_watcher/shared/getter/getter_suite_test.go diff --git a/pkg/omni/shared/getter/getter_test.go b/pkg/contract_watcher/shared/getter/getter_test.go similarity index 89% rename from pkg/omni/shared/getter/getter_test.go rename to pkg/contract_watcher/shared/getter/getter_test.go index 730767db..0f5d7cb6 100644 --- a/pkg/omni/shared/getter/getter_test.go +++ b/pkg/contract_watcher/shared/getter/getter_test.go @@ -22,21 +22,22 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/getter" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/getter" + "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("Interface Getter", func() { Describe("GetAbi", func() { It("Constructs and returns a custom abi based on results from supportsInterface calls", func() { expectedABI := `[` + constants.AddrChangeInterface + `,` + constants.NameChangeInterface + `,` + constants.ContentChangeInterface + `,` + constants.AbiChangeInterface + `,` + constants.PubkeyChangeInterface + `]` - + con := test_config.InfuraClient + infuraIPC := con.IPCPath blockNumber := int64(6885696) - infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" rawRpcClient, err := rpc.Dial(infuraIPC) Expect(err).NotTo(HaveOccurred()) rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) diff --git a/pkg/omni/shared/getter/interface_getter.go b/pkg/contract_watcher/shared/getter/interface_getter.go similarity index 96% rename from pkg/omni/shared/getter/interface_getter.go rename to pkg/contract_watcher/shared/getter/interface_getter.go index 70fc8273..8aee7d03 100644 --- a/pkg/omni/shared/getter/interface_getter.go +++ b/pkg/contract_watcher/shared/getter/interface_getter.go @@ -17,9 +17,9 @@ package getter import ( + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/fetcher" ) type InterfaceGetter interface { diff --git a/pkg/omni/shared/helpers/helpers.go b/pkg/contract_watcher/shared/helpers/helpers.go similarity index 100% rename from pkg/omni/shared/helpers/helpers.go rename to pkg/contract_watcher/shared/helpers/helpers.go diff --git a/pkg/omni/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go similarity index 87% rename from pkg/omni/shared/helpers/test_helpers/database.go rename to pkg/contract_watcher/shared/helpers/test_helpers/database.go index fb12cb3c..93c79459 100644 --- a/pkg/omni/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -24,6 +24,9 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -31,9 +34,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/test_config" ) type TransferLog struct { @@ -106,31 +107,17 @@ type Owner struct { Address string `db:"returned"` } -// TODO: consider whether this should be moved to libraries/shared -func SetupBC() core.BlockChain { - infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" - rawRpcClient, err := rpc.Dial(infuraIPC) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) - ethClient := ethclient.NewClient(rawRpcClient) - blockChainClient := client.NewEthClient(ethClient) - blockChainNode := node.MakeNode(rpcClient) - transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter) - - return blockChain -} - func SetupDBandBC() (*postgres.DB, core.BlockChain) { - infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" + con := test_config.InfuraClient + infuraIPC := con.IPCPath rawRpcClient, err := rpc.Dial(infuraIPC) Expect(err).NotTo(HaveOccurred()) rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) ethClient := ethclient.NewClient(rawRpcClient) blockChainClient := client.NewEthClient(ethClient) - blockChainNode := node.MakeNode(rpcClient) + node := node.MakeNode(rpcClient) transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter) + blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -181,7 +168,6 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract Abi: p.Abi(), ParsedAbi: p.ParsedAbi(), StartingBlock: 6194634, - LastBlock: 6507323, Events: p.GetEvents(wantedEvents), Methods: p.GetSelectMethods(wantedMethods), MethodArgs: map[string]bool{}, @@ -189,7 +175,6 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract }.Init() } -// TODO: consider whether this can be moved to plugin or libraries/shared func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) { db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -229,7 +214,6 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract { Abi: p.Abi(), ParsedAbi: p.ParsedAbi(), StartingBlock: 6194634, - LastBlock: 6507323, Events: p.GetEvents(wantedEvents), Methods: p.GetSelectMethods(wantedMethods), MethodArgs: map[string]bool{}, diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go similarity index 88% rename from pkg/omni/shared/helpers/test_helpers/mocks/entities.go rename to pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go index 9a2800c5..f9224847 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go @@ -18,14 +18,16 @@ package mocks import ( "encoding/json" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var TransferBlock1 = core.Block{ @@ -181,6 +183,13 @@ var MockHeader3 = core.Header{ Timestamp: "50000030", } +var MockHeader4 = core.Header{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + BlockNumber: 6194635, + Raw: rawFakeHeader, + Timestamp: "50000030", +} + var MockTransferLog1 = types.Log{ Index: 1, Address: common.HexToAddress(constants.TusdContractAddress), @@ -249,3 +258,50 @@ var MockNewOwnerLog2 = types.Log{ }, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000af21"), } + +var ens = strings.ToLower(constants.EnsContractAddress) +var tusd = strings.ToLower(constants.TusdContractAddress) + +var MockConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + "0x1234567890abcdef": true, + }, + Abis: map[string]string{ + "0x1234567890abcdef": "fake_abi", + }, + Events: map[string][]string{ + "0x1234567890abcdef": []string{"Transfer"}, + }, + Methods: map[string][]string{ + "0x1234567890abcdef": nil, + }, + MethodArgs: map[string][]string{ + "0x1234567890abcdef": nil, + }, + EventArgs: map[string][]string{ + "0x1234567890abcdef": nil, + }, +} + +var MockEmptyConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + "0x1234567890abcdef": true, + }, + Abis: map[string]string{ + "0x1234567890abcdef": "fake_abi", + }, + Events: map[string][]string{ + "0x1234567890abcdef": nil, + }, + Methods: map[string][]string{ + "0x1234567890abcdef": nil, + }, + MethodArgs: map[string][]string{ + "0x1234567890abcdef": nil, + }, + EventArgs: map[string][]string{ + "0x1234567890abcdef": nil, + }, +} diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/parser.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go similarity index 98% rename from pkg/omni/shared/helpers/test_helpers/mocks/parser.go rename to pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go index 5dd5d2e8..6ea0948d 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/parser.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go @@ -19,8 +19,8 @@ package mocks import ( "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Mock parser diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/test_data.go b/pkg/contract_watcher/shared/helpers/test_helpers/test_data.go new file mode 100644 index 00000000..c52fd20c --- /dev/null +++ b/pkg/contract_watcher/shared/helpers/test_helpers/test_data.go @@ -0,0 +1,109 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test_helpers + +import ( + "strings" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" +) + +var ens = strings.ToLower(constants.EnsContractAddress) +var tusd = strings.ToLower(constants.TusdContractAddress) + +var TusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + tusd: true, + }, + Abis: map[string]string{ + tusd: "", + }, + Events: map[string][]string{ + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + tusd: nil, + }, + MethodArgs: map[string][]string{ + tusd: nil, + }, + EventArgs: map[string][]string{ + tusd: nil, + }, + StartingBlocks: map[string]int64{ + tusd: 5197514, + }, +} + +var ENSConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + }, + Abis: map[string]string{ + ens: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + }, + Methods: map[string][]string{ + ens: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + }, + StartingBlocks: map[string]int64{ + ens: 3327417, + }, +} + +var ENSandTusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + tusd: true, + }, + Abis: map[string]string{ + ens: "", + tusd: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + ens: nil, + tusd: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, + StartingBlocks: map[string]int64{ + ens: 3327417, + tusd: 5197514, + }, +} diff --git a/pkg/omni/shared/parser/parser.go b/pkg/contract_watcher/shared/parser/parser.go similarity index 96% rename from pkg/omni/shared/parser/parser.go rename to pkg/contract_watcher/shared/parser/parser.go index 22686bfb..77af237b 100644 --- a/pkg/omni/shared/parser/parser.go +++ b/pkg/contract_watcher/shared/parser/parser.go @@ -22,9 +22,9 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Parser is used to fetch and parse contract ABIs @@ -98,7 +98,7 @@ func (p *parser) lookUp(contractAddr string) (string, error) { return v, nil } - return "", errors.New("ABI not present in lookup tabe") + return "", errors.New("ABI not present in lookup table") } // Returns only specified methods, if they meet the criteria diff --git a/pkg/omni/shared/parser/parser_suite_test.go b/pkg/contract_watcher/shared/parser/parser_suite_test.go similarity index 100% rename from pkg/omni/shared/parser/parser_suite_test.go rename to pkg/contract_watcher/shared/parser/parser_suite_test.go diff --git a/pkg/omni/shared/parser/parser_test.go b/pkg/contract_watcher/shared/parser/parser_test.go similarity index 95% rename from pkg/omni/shared/parser/parser_test.go rename to pkg/contract_watcher/shared/parser/parser_test.go index a3ee5bfc..95b065cb 100644 --- a/pkg/omni/shared/parser/parser_test.go +++ b/pkg/contract_watcher/shared/parser/parser_test.go @@ -21,11 +21,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Parser", func() { diff --git a/pkg/omni/shared/poller/poller.go b/pkg/contract_watcher/shared/poller/poller.go similarity index 93% rename from pkg/omni/shared/poller/poller.go rename to pkg/contract_watcher/shared/poller/poller.go index 7964e8bf..b3f1c51b 100644 --- a/pkg/omni/shared/poller/poller.go +++ b/pkg/contract_watcher/shared/poller/poller.go @@ -26,15 +26,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) type Poller interface { - PollContract(con contract.Contract) error + PollContract(con contract.Contract, lastBlock int64) error PollContractAt(con contract.Contract, blockNumber int64) error FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error } @@ -52,9 +52,11 @@ func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *po } } -func (p *poller) PollContract(con contract.Contract) error { - for i := con.StartingBlock; i <= con.LastBlock; i++ { - p.PollContractAt(con, i) +func (p *poller) PollContract(con contract.Contract, lastBlock int64) error { + for i := con.StartingBlock; i <= lastBlock; i++ { + if err := p.PollContractAt(con, i); err != nil { + return err + } } return nil @@ -98,7 +100,6 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error { if err != nil { return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } - strOut, err := stringify(out) if err != nil { return err @@ -138,8 +139,8 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { if len(args) == 0 { // If we haven't collected any args by now we can't call the method return nil } - results := make([]types.Result, 0, len(args)) + results := make([]types.Result, 0, len(args)) for arg := range args { in := []interface{}{arg} strIn := []interface{}{contract.StringifyArg(arg)} @@ -160,7 +161,6 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { result.Output = strOut results = append(results, result) } - // Persist result set as batch err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) if err != nil { @@ -204,7 +204,6 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { } results := make([]types.Result, 0, len(firstArgs)*len(secondArgs)) - for arg1 := range firstArgs { for arg2 := range secondArgs { in := []interface{}{arg1, arg2} @@ -215,18 +214,15 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { if err != nil { return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } - strOut, err := stringify(out) if err != nil { return err } - p.cache(out) result.Output = strOut result.Inputs = strIn results = append(results, result) - } } @@ -243,9 +239,8 @@ func (p *poller) FetchContractData(contractAbi, contractAddress, method string, return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber) } -// This is used to cache an method return value if method piping is turned on +// This is used to cache a method return value if method piping is turned on func (p *poller) cache(out interface{}) { - // Cache returned value if piping is turned on if p.contract.Piping { switch out.(type) { case common.Hash: diff --git a/pkg/omni/shared/repository/event_repository.go b/pkg/contract_watcher/shared/repository/event_repository.go similarity index 98% rename from pkg/omni/shared/repository/event_repository.go rename to pkg/contract_watcher/shared/repository/event_repository.go index a08849b4..14959a49 100644 --- a/pkg/omni/shared/repository/event_repository.go +++ b/pkg/contract_watcher/shared/repository/event_repository.go @@ -23,9 +23,9 @@ import ( "github.com/hashicorp/golang-lru" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) const ( diff --git a/pkg/omni/shared/repository/event_repository_test.go b/pkg/contract_watcher/shared/repository/event_repository_test.go similarity index 94% rename from pkg/omni/shared/repository/event_repository_test.go rename to pkg/contract_watcher/shared/repository/event_repository_test.go index aee6fe40..44a4cd66 100644 --- a/pkg/omni/shared/repository/event_repository_test.go +++ b/pkg/contract_watcher/shared/repository/event_repository_test.go @@ -26,17 +26,17 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + fc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" + lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + sr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - fc "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - lc "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" - lr "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" - sr "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Repository", func() { @@ -134,7 +134,8 @@ var _ = Describe("Repository", func() { Describe("PersistLogs", func() { BeforeEach(func() { - c := fc.NewConverter(con) + c := fc.Converter{} + c.Update(con) log, err = c.Convert(mockEvent, event) Expect(err).ToNot(HaveOccurred()) }) @@ -276,7 +277,8 @@ var _ = Describe("Repository", func() { headerRepository := repositories.NewHeaderRepository(db) headerID, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) Expect(err).ToNot(HaveOccurred()) - c := lc.NewConverter(con) + c := lc.Converter{} + c.Update(con) logs, err = c.Convert([]geth.Log{mockLog1, mockLog2}, event, headerID) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/omni/shared/repository/method_repository.go b/pkg/contract_watcher/shared/repository/method_repository.go similarity index 99% rename from pkg/omni/shared/repository/method_repository.go rename to pkg/contract_watcher/shared/repository/method_repository.go index e916e045..2616675e 100644 --- a/pkg/omni/shared/repository/method_repository.go +++ b/pkg/contract_watcher/shared/repository/method_repository.go @@ -23,8 +23,8 @@ import ( "github.com/hashicorp/golang-lru" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) const methodCacheSize = 1000 diff --git a/pkg/omni/shared/repository/method_repository_test.go b/pkg/contract_watcher/shared/repository/method_repository_test.go similarity index 95% rename from pkg/omni/shared/repository/method_repository_test.go rename to pkg/contract_watcher/shared/repository/method_repository_test.go index 54a665c2..b50f1ee5 100644 --- a/pkg/omni/shared/repository/method_repository_test.go +++ b/pkg/contract_watcher/shared/repository/method_repository_test.go @@ -23,12 +23,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Repository", func() { diff --git a/pkg/omni/shared/repository/repository_suite_test.go b/pkg/contract_watcher/shared/repository/repository_suite_test.go similarity index 100% rename from pkg/omni/shared/repository/repository_suite_test.go rename to pkg/contract_watcher/shared/repository/repository_suite_test.go diff --git a/pkg/omni/shared/retriever/address_retriever.go b/pkg/contract_watcher/shared/retriever/address_retriever.go similarity index 96% rename from pkg/omni/shared/retriever/address_retriever.go rename to pkg/contract_watcher/shared/retriever/address_retriever.go index 61e2b939..6acdf72b 100644 --- a/pkg/omni/shared/retriever/address_retriever.go +++ b/pkg/contract_watcher/shared/retriever/address_retriever.go @@ -18,14 +18,14 @@ package retriever import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "strings" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" ) // Address retriever is used to retrieve the addresses associated with a contract diff --git a/pkg/omni/shared/retriever/address_retriever_test.go b/pkg/contract_watcher/shared/retriever/address_retriever_test.go similarity index 86% rename from pkg/omni/shared/retriever/address_retriever_test.go rename to pkg/contract_watcher/shared/retriever/address_retriever_test.go index fd19a8f1..c8aa6fff 100644 --- a/pkg/omni/shared/retriever/address_retriever_test.go +++ b/pkg/contract_watcher/shared/retriever/address_retriever_test.go @@ -21,15 +21,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var mockEvent = core.WatchedEvent{ @@ -64,7 +64,8 @@ var _ = Describe("Address Retriever Test", func() { err = info.GenerateFilters() Expect(err).ToNot(HaveOccurred()) - c := converter.NewConverter(info) + c := converter.Converter{} + c.Update(info) log, err = c.Convert(mockEvent, event) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/omni/shared/retriever/retriever_suite_test.go b/pkg/contract_watcher/shared/retriever/retriever_suite_test.go similarity index 100% rename from pkg/omni/shared/retriever/retriever_suite_test.go rename to pkg/contract_watcher/shared/retriever/retriever_suite_test.go diff --git a/pkg/omni/shared/types/event.go b/pkg/contract_watcher/shared/types/event.go similarity index 98% rename from pkg/omni/shared/types/event.go rename to pkg/contract_watcher/shared/types/event.go index 7033f4ed..2818cc8d 100644 --- a/pkg/omni/shared/types/event.go +++ b/pkg/contract_watcher/shared/types/event.go @@ -38,7 +38,7 @@ type Field struct { // Struct to hold instance of an event log data type Log struct { - Id int64 // VulcanizeIdLog for full sync and header ID for light sync omni watcher + Id int64 // VulcanizeIdLog for full sync and header ID for light sync contract watcher Values map[string]string // Map of event input names to their values // Used for full sync only diff --git a/pkg/omni/shared/types/method.go b/pkg/contract_watcher/shared/types/method.go similarity index 100% rename from pkg/omni/shared/types/method.go rename to pkg/contract_watcher/shared/types/method.go diff --git a/pkg/omni/shared/types/mode.go b/pkg/contract_watcher/shared/types/mode.go similarity index 88% rename from pkg/omni/shared/types/mode.go rename to pkg/contract_watcher/shared/types/mode.go index 62057a16..9f37a857 100644 --- a/pkg/omni/shared/types/mode.go +++ b/pkg/contract_watcher/shared/types/mode.go @@ -47,7 +47,7 @@ func (mode Mode) MarshalText() ([]byte, error) { case FullSync: return []byte("full"), nil default: - return nil, fmt.Errorf("omni watcher: unknown mode %d, want LightSync or FullSync", mode) + return nil, fmt.Errorf("contract watcher: unknown mode %d, want LightSync or FullSync", mode) } } @@ -58,7 +58,7 @@ func (mode *Mode) UnmarshalText(text []byte) error { case "full": *mode = FullSync default: - return fmt.Errorf(`omni watcher: unknown mode %q, want "light" or "full"`, text) + return fmt.Errorf(`contract watcher: unknown mode %q, want "light" or "full"`, text) } return nil } diff --git a/pkg/fakes/mock_parser.go b/pkg/fakes/mock_parser.go index cac8dd9c..dd03bbab 100644 --- a/pkg/fakes/mock_parser.go +++ b/pkg/fakes/mock_parser.go @@ -2,7 +2,7 @@ package fakes import ( "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) type MockParser struct { @@ -15,8 +15,9 @@ func (*MockParser) Parse(contractAddr string) error { return nil } -func (*MockParser) ParseAbiStr(abiStr string) error { - panic("implement me") +func (m *MockParser) ParseAbiStr(abiStr string) error { + m.AbiToReturn = abiStr + return nil } func (parser *MockParser) Abi() string { diff --git a/pkg/fakes/mock_poller.go b/pkg/fakes/mock_poller.go index f1a1caec..2d782b18 100644 --- a/pkg/fakes/mock_poller.go +++ b/pkg/fakes/mock_poller.go @@ -1,14 +1,14 @@ package fakes import ( - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" ) type MockPoller struct { ContractName string } -func (*MockPoller) PollContract(con contract.Contract) error { +func (*MockPoller) PollContract(con contract.Contract, lastBlock int64) error { panic("implement me") } diff --git a/pkg/geth/contract.go b/pkg/geth/contract.go index f59cab5f..b5f968ec 100644 --- a/pkg/geth/contract.go +++ b/pkg/geth/contract.go @@ -43,7 +43,11 @@ func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, if err != nil { return err } - output, err := blockChain.callContract(address, input, big.NewInt(blockNumber)) + var bn *big.Int + if blockNumber > 0 { + bn = big.NewInt(blockNumber) + } + output, err := blockChain.callContract(address, input, bn) if err != nil { return err } diff --git a/pkg/omni/full/transformer/transformer.go b/pkg/omni/full/transformer/transformer.go deleted file mode 100644 index 3a37c084..00000000 --- a/pkg/omni/full/transformer/transformer.go +++ /dev/null @@ -1,271 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package transformer - -import ( - "errors" - "strings" - - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" -) - -// Requires a fully synced vDB and a running eth node (or infura) -type Transformer struct { - // Database interfaces - datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() - datastore.WatchedEventRepository // Watched event log views, created by the log filters - repository.EventRepository // Holds transformed watched event log data - - // Pre-processing interfaces - parser.Parser // Parses events and methods out of contract abi fetched using contract address - retriever.BlockRetriever // Retrieves first block for contract and current block height - - // Processing interfaces - converter.Converter // Converts watched event logs into custom log - poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore - - // Ethereum network name; default "" is mainnet - Network string - - // Store contract info as mapping to contract address - Contracts map[string]*contract.Contract - - // Targeted subset of events/methods - // Stored as map sof contract address to events/method names of interest - WatchedEvents map[string][]string // Default/empty event list means all are watched - WantedMethods map[string][]string // Default/empty method list means none are polled - - // Starting block for contracts - ContractStart map[string]int64 - - // Lists of addresses to filter event or method data - // before persisting; if empty no filter is applied - EventArgs map[string][]string - MethodArgs map[string][]string - - // Whether or not to create a list of emitted address or hashes for the contract in postgres - CreateAddrList map[string]bool - CreateHashList map[string]bool - - // Method piping on/off for a contract - Piping map[string]bool -} - -// Transformer takes in config for blockchain, database, and network id -func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer { - return &Transformer{ - Poller: poller.NewPoller(BC, DB, types.FullSync), - Parser: parser.NewParser(network), - BlockRetriever: retriever.NewBlockRetriever(DB), - Converter: converter.NewConverter(&contract.Contract{}), - Contracts: map[string]*contract.Contract{}, - WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, - FilterRepository: repositories.FilterRepository{DB: DB}, - EventRepository: repository.NewEventRepository(DB, types.FullSync), - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - Piping: map[string]bool{}, - } -} - -// Use after creating and setting transformer -// Loops over all of the addr => filter sets -// Uses parser to pull event info from abi -// Use this info to generate event filters -func (transformer *Transformer) Init() error { - for contractAddr, subset := range transformer.WatchedEvents { - // Get Abi - err := transformer.Parser.Parse(contractAddr) - if err != nil { - return err - } - - // Get first block and most recent block number in the header repo - firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr) - if err != nil { - return err - } - lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock() - if err != nil { - return err - } - - // Set to specified range if it falls within the bounds - if firstBlock < transformer.ContractStart[contractAddr] { - firstBlock = transformer.ContractStart[contractAddr] - } - - // Get contract name if it has one - var name = new(string) - transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock) - - // Remove any potential accidental duplicate inputs in arg filter values - eventArgs := map[string]bool{} - for _, arg := range transformer.EventArgs[contractAddr] { - eventArgs[arg] = true - } - methodArgs := map[string]bool{} - for _, arg := range transformer.MethodArgs[contractAddr] { - methodArgs[arg] = true - } - - // Aggregate info into contract object - info := contract.Contract{ - Name: *name, - Network: transformer.Network, - Address: contractAddr, - Abi: transformer.Parser.Abi(), - ParsedAbi: transformer.Parser.ParsedAbi(), - StartingBlock: firstBlock, - LastBlock: lastBlock, - Events: transformer.Parser.GetEvents(subset), - Methods: transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]), - FilterArgs: eventArgs, - MethodArgs: methodArgs, - CreateAddrList: transformer.CreateAddrList[contractAddr], - CreateHashList: transformer.CreateHashList[contractAddr], - Piping: transformer.Piping[contractAddr], - }.Init() - - // Use info to create filters - err = info.GenerateFilters() - if err != nil { - return err - } - - // Iterate over filters and push them to the repo using filter repository interface - for _, filter := range info.Filters { - err = transformer.FilterRepository.CreateFilter(filter) - if err != nil { - return err - } - } - - // Store contract info for further processing - transformer.Contracts[contractAddr] = info - } - - return nil -} - -// Iterates through stored, initialized contract objects -// Iterates through contract's event filters, grabbing watched event logs -// Uses converter to convert logs into custom log type -// Persists converted logs into custuom postgres tables -// Calls selected methods, using token holder address generated during event log conversion -func (transformer Transformer) Execute() error { - if len(transformer.Contracts) == 0 { - return errors.New("error: transformer has no initialized contracts to work with") - } - // Iterate through all internal contracts - for _, con := range transformer.Contracts { - // Update converter with current contract - transformer.Update(con) - - // Iterate through contract filters and get watched event logs - for eventSig, filter := range con.Filters { - watchedEvents, err := transformer.GetWatchedEvents(filter.Name) - if err != nil { - return err - } - - // Iterate over watched event logs - for _, we := range watchedEvents { - // Convert them to our custom log type - cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig]) - if err != nil { - return err - } - if cstm == nil { - continue - } - - // If log is not empty, immediately persist in repo - // Run this in seperate goroutine? - err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) - if err != nil { - return err - } - } - } - - // After persisting all watched event logs - // poller polls select contract methods - // and persists the results into custom pg tables - // Run this in seperate goroutine? - if err := transformer.PollContract(*con); err != nil { - return err - } - } - - return nil -} - -// Used to set which contract addresses and which of their events to watch -func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) { - transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to watch events for -func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) { - transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set which contract addresses and which of their methods to call -func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) { - transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to poll methods on -func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) { - transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set the block range to watch for a given address -func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) { - transformer.ContractStart[strings.ToLower(contractAddr)] = start -} - -// Used to set whether or not to persist an account address list -func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) { - transformer.CreateAddrList[strings.ToLower(contractAddr)] = on -} - -// Used to set whether or not to persist an hash list -func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) { - transformer.CreateHashList[strings.ToLower(contractAddr)] = on -} - -// Used to turn method piping on for a contract -func (transformer *Transformer) SetPiping(contractAddr string, on bool) { - transformer.Piping[strings.ToLower(contractAddr)] = on -} diff --git a/pkg/omni/full/transformer/transformer_test.go b/pkg/omni/full/transformer/transformer_test.go deleted file mode 100644 index f25ca8f6..00000000 --- a/pkg/omni/full/transformer/transformer_test.go +++ /dev/null @@ -1,176 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package transformer_test - -import ( - "math/rand" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" -) - -var _ = Describe("Transformer", func() { - var fakeAddress = "0x1234567890abcdef" - rand.Seed(time.Now().UnixNano()) - - Describe("SetEvents", func() { - It("Sets which events to watch from the given contract address", func() { - watchedEvents := []string{"Transfer", "Mint"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, watchedEvents) - Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents)) - }) - }) - - Describe("SetEventAddrs", func() { - It("Sets which account addresses to watch events for", func() { - eventAddrs := []string{"test1", "test2"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEventArgs(fakeAddress, eventAddrs) - Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs)) - }) - }) - - Describe("SetMethods", func() { - It("Sets which methods to poll at the given contract address", func() { - watchedMethods := []string{"balanceOf", "totalSupply"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethods(fakeAddress, watchedMethods) - Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods)) - }) - }) - - Describe("SetMethodAddrs", func() { - It("Sets which account addresses to poll methods against", func() { - methodAddrs := []string{"test1", "test2"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethodArgs(fakeAddress, methodAddrs) - Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs)) - }) - }) - - Describe("SetStartingBlock", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetStartingBlock(fakeAddress, 11) - Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11))) - }) - }) - - Describe("SetCreateAddrList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateAddrList(fakeAddress, true) - Expect(t.CreateAddrList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("SetCreateHashList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateHashList(fakeAddress, true) - Expect(t.CreateHashList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("Init", func() { - It("Initializes transformer's contract objects", func() { - blockRetriever := &fakes.MockFullBlockRetriever{} - firstBlock := int64(1) - mostRecentBlock := int64(2) - blockRetriever.FirstBlock = firstBlock - blockRetriever.MostRecentBlock = mostRecentBlock - - parsr := &fakes.MockParser{} - fakeAbi := "fake_abi" - eventName := "Transfer" - event := types.Event{} - parsr.AbiToReturn = fakeAbi - parsr.EventName = eventName - parsr.Event = event - - pollr := &fakes.MockPoller{} - fakeContractName := "fake_contract_name" - pollr.ContractName = fakeContractName - - t := getTransformer(blockRetriever, parsr, pollr) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - - Expect(err).ToNot(HaveOccurred()) - - c, ok := t.Contracts[fakeAddress] - Expect(ok).To(Equal(true)) - - Expect(c.StartingBlock).To(Equal(firstBlock)) - Expect(c.LastBlock).To(Equal(mostRecentBlock)) - Expect(c.Abi).To(Equal(fakeAbi)) - Expect(c.Name).To(Equal(fakeContractName)) - Expect(c.Address).To(Equal(fakeAddress)) - }) - - It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { - blockRetriever := &fakes.MockFullBlockRetriever{} - blockRetriever.FirstBlockErr = fakes.FakeError - t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("Does nothing if watched events are unset", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - - err := t.Init() - - Expect(err).ToNot(HaveOccurred()) - - _, ok := t.Contracts[fakeAddress] - Expect(ok).To(Equal(false)) - }) - }) -}) - -func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { - return transformer.Transformer{ - FilterRepository: &fakes.MockFilterRepository{}, - Parser: parsr, - BlockRetriever: blockRetriever, - Poller: pollr, - Contracts: map[string]*contract.Contract{}, - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - } -} diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/omni/light/transformer/transformer_test.go deleted file mode 100644 index 31e3c194..00000000 --- a/pkg/omni/light/transformer/transformer_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package transformer_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" -) - -var _ = Describe("Transformer", func() { - var fakeAddress = "0x1234567890abcdef" - - Describe("SetEvents", func() { - It("Sets which events to watch from the given contract address", func() { - watchedEvents := []string{"Transfer", "Mint"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, watchedEvents) - Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents)) - }) - }) - - Describe("SetEventAddrs", func() { - It("Sets which account addresses to watch events for", func() { - eventAddrs := []string{"test1", "test2"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEventArgs(fakeAddress, eventAddrs) - Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs)) - }) - }) - - Describe("SetMethods", func() { - It("Sets which methods to poll at the given contract address", func() { - watchedMethods := []string{"balanceOf", "totalSupply"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethods(fakeAddress, watchedMethods) - Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods)) - }) - }) - - Describe("SetMethodAddrs", func() { - It("Sets which account addresses to poll methods against", func() { - methodAddrs := []string{"test1", "test2"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethodArgs(fakeAddress, methodAddrs) - Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs)) - }) - }) - - Describe("SetStartingBlock", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetStartingBlock(fakeAddress, 11) - Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11))) - }) - }) - - Describe("SetCreateAddrList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateAddrList(fakeAddress, true) - Expect(t.CreateAddrList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("SetCreateHashList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateHashList(fakeAddress, true) - Expect(t.CreateHashList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("Init", func() { - It("Initializes transformer's contract objects", func() { - blockRetriever := &fakes.MockLightBlockRetriever{} - firstBlock := int64(1) - blockRetriever.FirstBlock = firstBlock - - parsr := &fakes.MockParser{} - fakeAbi := "fake_abi" - parsr.AbiToReturn = fakeAbi - - pollr := &fakes.MockPoller{} - fakeContractName := "fake_contract_name" - pollr.ContractName = fakeContractName - - t := getFakeTransformer(blockRetriever, parsr, pollr) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - - Expect(err).ToNot(HaveOccurred()) - - c, ok := t.Contracts[fakeAddress] - Expect(ok).To(Equal(true)) - - Expect(c.StartingBlock).To(Equal(firstBlock)) - Expect(c.LastBlock).To(Equal(int64(-1))) - Expect(c.Abi).To(Equal(fakeAbi)) - Expect(c.Name).To(Equal(fakeContractName)) - Expect(c.Address).To(Equal(fakeAddress)) - }) - - It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() { - blockRetriever := &fakes.MockLightBlockRetriever{} - blockRetriever.FirstBlockErr = fakes.FakeError - t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) - }) - - It("Does nothing if watched events are unset", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - - err := t.Init() - - Expect(err).ToNot(HaveOccurred()) - - _, ok := t.Contracts[fakeAddress] - Expect(ok).To(Equal(false)) - }) - }) - -}) - -func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { - return transformer.Transformer{ - Parser: parsr, - BlockRetriever: blockRetriever, - Poller: pollr, - HeaderRepository: &fakes.MockLightHeaderRepository{}, - Contracts: map[string]*contract.Contract{}, - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - } -} diff --git a/pkg/omni/shared/transformer/interface.go b/pkg/omni/shared/transformer/interface.go deleted file mode 100644 index ef0d331b..00000000 --- a/pkg/omni/shared/transformer/interface.go +++ /dev/null @@ -1,32 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package transformer - -// Used to extract any/all events and a subset of method (state variable) -// data for any contract and persists it to custom postgres tables in vDB -type Transformer interface { - SetEvents(contractAddr string, filterSet []string) - SetEventArgs(contractAddr string, filterSet []string) - SetMethods(contractAddr string, filterSet []string) - SetMethodArgs(contractAddr string, filterSet []string) - SetStartingBlock(contractAddr string, start int64) - SetCreateAddrList(contractAddr string, on bool) - SetCreateHashList(contractAddr string, on bool) - SetPiping(contractAddr string, on bool) - Init() error - Execute() error -} diff --git a/pkg/plugin/test_helpers/database.go b/pkg/plugin/test_helpers/database.go index 0ab5c63f..345683f0 100644 --- a/pkg/plugin/test_helpers/database.go +++ b/pkg/plugin/test_helpers/database.go @@ -54,7 +54,7 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) { } func TearDown(db *postgres.DB) { - tx, err := db.Begin() + tx, err := db.Beginx() Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM headers`) diff --git a/pkg/plugin/writer/writer.go b/pkg/plugin/writer/writer.go index 6c0b4ffd..e888bc5b 100644 --- a/pkg/plugin/writer/writer.go +++ b/pkg/plugin/writer/writer.go @@ -74,13 +74,17 @@ func (w *writer) WritePlugin() error { f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List( Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "EventTransformerInitializer"), Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"), + Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "ContractTransformerInitializer"), )).Block(Return( Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "EventTransformerInitializer").Values(code[config.EthEvent]...), Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers + "StorageTransformerInitializer").Values(code[config.EthStorage]...), + Index().Qual( + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", + "ContractTransformerInitializer").Values(code[config.EthContract]...))) // Exports the collected event and storage transformer initializers // Write code to destination file err = f.Save(goFile) @@ -100,6 +104,8 @@ func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "EventTransformerInitializer")) case config.EthStorage: code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer")) + case config.EthContract: + code[config.EthContract] = append(code[config.EthContract], Qual(path, "ContractTransformerInitializer")) default: return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type)) } diff --git a/test_config/test_config.go b/test_config/test_config.go index b64a9278..0f0071ae 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -17,6 +17,7 @@ package test_config import ( + "errors" "fmt" "os" @@ -48,10 +49,10 @@ func setTestConfig() { TestConfig.SetConfigName("private") TestConfig.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/") err := TestConfig.ReadInConfig() - ipc := TestConfig.GetString("client.ipcPath") if err != nil { log.Fatal(err) } + ipc := TestConfig.GetString("client.ipcPath") hn := TestConfig.GetString("database.hostname") port := TestConfig.GetInt("database.port") name := TestConfig.GetString("database.name") @@ -71,10 +72,20 @@ func setInfuraConfig() { Infura.SetConfigName("infura") Infura.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/") err := Infura.ReadInConfig() - ipc := Infura.GetString("client.ipcpath") if err != nil { log.Fatal(err) } + ipc := Infura.GetString("client.ipcpath") + + // If we don't have an ipc path in the config file, check the env variable + if ipc == "" { + Infura.BindEnv("url", "INFURA_URL") + ipc = Infura.GetString("url") + } + if ipc == "" { + log.Fatal(errors.New("infura.toml IPC path or $INFURA_URL env variable need to be set")) + } + InfuraClient = config.Client{ IPCPath: ipc, }