From 975f13b969b38340bfc1fcbd594b6eee4e74f117 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 23 Nov 2018 12:12:24 -0600 Subject: [PATCH] reorganizing omni directory and beginning light watcher work --- cmd/erc20.go | 2 +- cmd/lightOmniWatcher.go | 126 ++++++++++ cmd/omniWatcher.go | 4 +- .../event_triggered/dai/converter.go | 4 +- .../event_triggered/dai/converter_test.go | 4 +- .../event_triggered/dai/integration_test.go | 2 +- .../event_triggered/dai/repository_test.go | 2 +- .../event_triggered/dai/transformer.go | 2 +- .../event_triggered/dai/transformer_test.go | 2 +- .../erc20_watcher/every_block/getter_test.go | 2 +- .../every_block/integration_test.go | 2 +- .../every_block/transformer_test.go | 2 +- examples/generic/config.go | 2 +- .../generic/event_triggered/tusd/converter.go | 4 +- .../event_triggered/tusd/converter_test.go | 4 +- .../event_triggered/tusd/integration_test.go | 2 +- .../event_triggered/tusd/repository_test.go | 2 +- .../event_triggered/tusd/transformer.go | 2 +- .../event_triggered/tusd/transformer_test.go | 2 +- examples/generic/every_block/getter_test.go | 2 +- examples/mocks/fetcher.go | 10 +- pkg/core/blockchain.go | 8 +- pkg/crypto/parser.go | 4 +- pkg/datastore/ethereum/level/database.go | 2 +- .../ethereum/level/database_reader.go | 20 +- pkg/fakes/mock_blockchain.go | 56 +++-- pkg/fakes/mock_level_database_reader.go | 4 +- .../converters/common/block_converter_test.go | 2 +- .../common/receipt_converter_test.go | 2 +- pkg/omni/{ => full}/converter/converter.go | 28 +-- .../converter/converter_suite_test.go | 2 +- .../{ => full}/converter/converter_test.go | 34 +-- .../{ => full}/repository/event_repository.go | 42 ++-- .../repository/event_repository_test.go | 16 +- .../repository/repository_suite_test.go | 2 +- .../{ => full}/retriever/address_retriever.go | 11 +- .../retriever/address_retriever_test.go | 18 +- .../{ => full}/retriever/block_retriever.go | 11 +- .../retriever/block_retriever_test.go | 6 +- .../retriever/retriever_suite_test.go | 2 +- .../{ => full}/transformer/transformer.go | 42 ++-- .../transformer/transformer_suite_test.go | 2 +- .../transformer/transformer_test.go | 70 +----- pkg/omni/light/converter/converter.go | 50 ++++ .../light/converter/converter_suite_test.go | 35 +++ pkg/omni/light/converter/converter_test.go | 17 ++ pkg/omni/light/fetcher/fetcher.go | 70 ++++++ pkg/omni/light/fetcher/fetcher_suite_test.go | 35 +++ pkg/omni/light/fetcher/fetcher_test.go | 66 +++++ pkg/omni/light/repository/event_repository.go | 181 +++++++++++++ .../light/repository/event_repository_test.go | 17 ++ .../light/repository/header_repository.go | 70 ++++++ .../repository/header_repository_test.go | 17 ++ .../light/repository/repository_suite_test.go | 35 +++ pkg/omni/light/retriever/block_retriever.go | 60 +++++ .../light/retriever/block_retriever_test.go | 78 ++++++ .../light/retriever/retriever_suite_test.go | 35 +++ pkg/omni/light/transformer/transformer.go | 238 ++++++++++++++++++ .../transformer/transformer_suite_test.go | 35 +++ .../light/transformer/transformer_test.go | 136 ++++++++++ pkg/omni/{ => shared}/constants/constants.go | 2 +- pkg/omni/{ => shared}/contract/contract.go | 20 +- .../contract/contract_suite_test.go | 0 .../{ => shared}/contract/contract_test.go | 4 +- pkg/omni/{ => shared}/helpers/helpers.go | 0 pkg/omni/{ => shared}/helpers/mocks/parser.go | 2 +- .../helpers/test_helpers/database.go | 29 +-- .../shared/helpers/test_helpers/entities.go | 128 ++++++++++ pkg/omni/{ => shared}/parser/parser.go | 2 +- .../{ => shared}/parser/parser_suite_test.go | 0 pkg/omni/{ => shared}/parser/parser_test.go | 6 +- pkg/omni/{ => shared}/poller/poller.go | 12 +- .../{ => shared}/poller/poller_suite_test.go | 0 pkg/omni/{ => shared}/poller/poller_test.go | 8 +- .../repository/method_repository.go | 26 +- .../repository/method_repository_test.go | 14 +- .../repository/repository_suite_test.go | 35 +++ pkg/omni/shared/transformer/interface.go | 29 +++ pkg/omni/shared/types/event.go | 98 ++++++++ .../entities.go => shared/types/method.go} | 66 ----- .../ethereum/go-ethereum/interfaces.go | 3 +- 81 files changed, 1844 insertions(+), 383 deletions(-) create mode 100644 cmd/lightOmniWatcher.go rename pkg/omni/{ => full}/converter/converter.go (81%) rename pkg/omni/{ => full}/converter/converter_suite_test.go (95%) rename pkg/omni/{ => full}/converter/converter_test.go (63%) rename pkg/omni/{ => full}/repository/event_repository.go (79%) rename pkg/omni/{ => full}/repository/event_repository_test.go (92%) rename pkg/omni/{ => full}/repository/repository_suite_test.go (95%) rename pkg/omni/{ => full}/retriever/address_retriever.go (95%) rename pkg/omni/{ => full}/retriever/address_retriever_test.go (87%) rename pkg/omni/{ => full}/retriever/block_retriever.go (97%) rename pkg/omni/{ => full}/retriever/block_retriever_test.go (97%) rename pkg/omni/{ => full}/retriever/retriever_suite_test.go (95%) rename pkg/omni/{ => full}/transformer/transformer.go (87%) rename pkg/omni/{ => full}/transformer/transformer_suite_test.go (95%) rename pkg/omni/{ => full}/transformer/transformer_test.go (72%) create mode 100644 pkg/omni/light/converter/converter.go create mode 100644 pkg/omni/light/converter/converter_suite_test.go create mode 100644 pkg/omni/light/converter/converter_test.go create mode 100644 pkg/omni/light/fetcher/fetcher.go create mode 100644 pkg/omni/light/fetcher/fetcher_suite_test.go create mode 100644 pkg/omni/light/fetcher/fetcher_test.go create mode 100644 pkg/omni/light/repository/event_repository.go create mode 100644 pkg/omni/light/repository/event_repository_test.go create mode 100644 pkg/omni/light/repository/header_repository.go create mode 100644 pkg/omni/light/repository/header_repository_test.go create mode 100644 pkg/omni/light/repository/repository_suite_test.go create mode 100644 pkg/omni/light/retriever/block_retriever.go create mode 100644 pkg/omni/light/retriever/block_retriever_test.go create mode 100644 pkg/omni/light/retriever/retriever_suite_test.go create mode 100644 pkg/omni/light/transformer/transformer.go create mode 100644 pkg/omni/light/transformer/transformer_suite_test.go create mode 100644 pkg/omni/light/transformer/transformer_test.go rename pkg/omni/{ => shared}/constants/constants.go (99%) rename pkg/omni/{ => shared}/contract/contract.go (80%) rename pkg/omni/{ => shared}/contract/contract_suite_test.go (100%) rename pkg/omni/{ => shared}/contract/contract_test.go (98%) rename pkg/omni/{ => shared}/helpers/helpers.go (100%) rename pkg/omni/{ => shared}/helpers/mocks/parser.go (98%) rename pkg/omni/{ => shared}/helpers/test_helpers/database.go (89%) create mode 100644 pkg/omni/shared/helpers/test_helpers/entities.go rename pkg/omni/{ => shared}/parser/parser.go (98%) rename pkg/omni/{ => shared}/parser/parser_suite_test.go (100%) rename pkg/omni/{ => shared}/parser/parser_test.go (97%) rename pkg/omni/{ => shared}/poller/poller.go (95%) rename pkg/omni/{ => shared}/poller/poller_suite_test.go (100%) rename pkg/omni/{ => shared}/poller/poller_test.go (94%) rename pkg/omni/{ => shared}/repository/method_repository.go (83%) rename pkg/omni/{ => shared}/repository/method_repository_test.go (89%) create mode 100644 pkg/omni/shared/repository/repository_suite_test.go create mode 100644 pkg/omni/shared/transformer/interface.go create mode 100644 pkg/omni/shared/types/event.go rename pkg/omni/{types/entities.go => shared/types/method.go} (65%) diff --git a/cmd/erc20.go b/cmd/erc20.go index 39d00376..5ca89d62 100644 --- a/cmd/erc20.go +++ b/cmd/erc20.go @@ -26,7 +26,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/every_block" "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/libraries/shared" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" "github.com/vulcanize/vulcanizedb/utils" ) diff --git a/cmd/lightOmniWatcher.go b/cmd/lightOmniWatcher.go new file mode 100644 index 00000000..4aae5bf3 --- /dev/null +++ b/cmd/lightOmniWatcher.go @@ -0,0 +1,126 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/vulcanize/vulcanizedb/libraries/shared" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" + "github.com/vulcanize/vulcanizedb/utils" +) + +// omniWatcherCmd represents the omniWatcher command +var lightOmniWatcherCmd = &cobra.Command{ + Use: "lightOmniWatcher", + Short: "Watches events at the provided contract address using lightSynced vDB", + Long: `Uses input contract address and event filters to watch events + +Expects an ethereum node to be running +Expects lightSync to have been run and the presence of headers in the Vulcanize database +Requires a .toml config file: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" +`, + Run: func(cmd *cobra.Command, args []string) { + lightOmniWatcher() + }, +} + +func lightOmniWatcher() { + if contractAddress == "" && len(contractAddresses) == 0 { + log.Fatal("Contract address required") + } + + if len(contractEvents) == 0 || len(contractMethods) == 0 { + var str string + for str != "y" { + reader := bufio.NewReader(os.Stdin) + if len(contractEvents) == 0 && len(contractMethods) == 0 { + fmt.Print("Warning: no events or methods specified.\n Proceed to watch every event and poll no methods? (Y/n)\n> ") + } else if len(contractEvents) == 0 { + fmt.Print("Warning: no events specified.\n Proceed to watch every event? (Y/n)\n> ") + } else { + fmt.Print("Warning: no methods specified.\n Proceed to poll no methods? (Y/n)\n> ") + } + resp, err := reader.ReadBytes('\n') + if err != nil { + log.Fatal(err) + } + + str = strings.ToLower(string(resp)) + if str == "n" { + return + } + } + } + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + blockChain := getBlockChain() + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + t := transformer.NewTransformer(network, blockChain, &db) + + contractAddresses = append(contractAddresses, contractAddress) + for _, addr := range contractAddresses { + t.SetEvents(addr, contractEvents) + t.SetMethods(addr, contractMethods) + t.SetEventAddrs(addr, eventAddrs) + t.SetMethodAddrs(addr, methodAddrs) + t.SetRange(addr, [2]int64{startingBlockNumber, endingBlockNumber}) + } + + err := t.Init() + if err != nil { + log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) + } + + w := shared.Watcher{} + w.AddTransformer(t) + + for range ticker.C { + w.Execute() + } +} + +func init() { + rootCmd.AddCommand(lightOmniWatcherCmd) + + omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") + omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address") + omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched") + omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled") + omniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses") + omniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses") + omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) + omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists") + omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block") +} diff --git a/cmd/omniWatcher.go b/cmd/omniWatcher.go index 6e293d7b..8c30268a 100644 --- a/cmd/omniWatcher.go +++ b/cmd/omniWatcher.go @@ -27,14 +27,14 @@ import ( "github.com/spf13/cobra" "github.com/vulcanize/vulcanizedb/libraries/shared" - "github.com/vulcanize/vulcanizedb/pkg/omni/transformer" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" "github.com/vulcanize/vulcanizedb/utils" ) // omniWatcherCmd represents the omniWatcher command var omniWatcherCmd = &cobra.Command{ Use: "omniWatcher", - Short: "Watches events at the provided contract address", + Short: "Watches events at the provided contract address using fully synced vDB", Long: `Uses input contract address and event filters to watch events Expects an ethereum node to be running diff --git a/examples/erc20_watcher/event_triggered/dai/converter.go b/examples/erc20_watcher/event_triggered/dai/converter.go index 62f3e37e..d5ecb9f9 100644 --- a/examples/erc20_watcher/event_triggered/dai/converter.go +++ b/examples/erc20_watcher/event_triggered/dai/converter.go @@ -24,8 +24,8 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) // Converter converts a raw event log into its corresponding entity diff --git a/examples/erc20_watcher/event_triggered/dai/converter_test.go b/examples/erc20_watcher/event_triggered/dai/converter_test.go index 12a89795..d303772f 100644 --- a/examples/erc20_watcher/event_triggered/dai/converter_test.go +++ b/examples/erc20_watcher/event_triggered/dai/converter_test.go @@ -25,8 +25,8 @@ import ( "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered/dai" "github.com/vulcanize/vulcanizedb/examples/generic" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) var expectedTransferModel = event_triggered.TransferModel{ diff --git a/examples/erc20_watcher/event_triggered/dai/integration_test.go b/examples/erc20_watcher/event_triggered/dai/integration_test.go index 9f2aeffd..d4e59f69 100644 --- a/examples/erc20_watcher/event_triggered/dai/integration_test.go +++ b/examples/erc20_watcher/event_triggered/dai/integration_test.go @@ -26,7 +26,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var transferLog = core.Log{ diff --git a/examples/erc20_watcher/event_triggered/dai/repository_test.go b/examples/erc20_watcher/event_triggered/dai/repository_test.go index c83bbdb4..12ace1e1 100644 --- a/examples/erc20_watcher/event_triggered/dai/repository_test.go +++ b/examples/erc20_watcher/event_triggered/dai/repository_test.go @@ -31,7 +31,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) var transferEntity = &dai.TransferEntity{ diff --git a/examples/erc20_watcher/event_triggered/dai/transformer.go b/examples/erc20_watcher/event_triggered/dai/transformer.go index 558b9a8c..fda12307 100644 --- a/examples/erc20_watcher/event_triggered/dai/transformer.go +++ b/examples/erc20_watcher/event_triggered/dai/transformer.go @@ -27,7 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) type ERC20EventTransformer struct { diff --git a/examples/erc20_watcher/event_triggered/dai/transformer_test.go b/examples/erc20_watcher/event_triggered/dai/transformer_test.go index 9ee27261..71da2806 100644 --- a/examples/erc20_watcher/event_triggered/dai/transformer_test.go +++ b/examples/erc20_watcher/event_triggered/dai/transformer_test.go @@ -23,7 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/erc20_watcher/event_triggered/dai" "github.com/vulcanize/vulcanizedb/examples/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var blockID1 = int64(5428074) diff --git a/examples/erc20_watcher/every_block/getter_test.go b/examples/erc20_watcher/every_block/getter_test.go index 19ce529d..ce409e7e 100644 --- a/examples/erc20_watcher/every_block/getter_test.go +++ b/examples/erc20_watcher/every_block/getter_test.go @@ -31,7 +31,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var _ = Describe("ERC20 Getter", func() { diff --git a/examples/erc20_watcher/every_block/integration_test.go b/examples/erc20_watcher/every_block/integration_test.go index 91181ccb..c9ee0183 100644 --- a/examples/erc20_watcher/every_block/integration_test.go +++ b/examples/erc20_watcher/every_block/integration_test.go @@ -30,7 +30,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) func setLastBlockOnChain(blockChain *fakes.MockBlockChain, blockNumber int64) { diff --git a/examples/erc20_watcher/every_block/transformer_test.go b/examples/erc20_watcher/every_block/transformer_test.go index 78da9e31..43a6dce1 100644 --- a/examples/erc20_watcher/every_block/transformer_test.go +++ b/examples/erc20_watcher/every_block/transformer_test.go @@ -30,7 +30,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/test_helpers" "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var testContractConfig = shared.ContractConfig{ diff --git a/examples/generic/config.go b/examples/generic/config.go index c5f43f41..a185980d 100644 --- a/examples/generic/config.go +++ b/examples/generic/config.go @@ -18,7 +18,7 @@ package generic import ( "github.com/vulcanize/vulcanizedb/libraries/shared" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var DaiConfig = shared.ContractConfig{ diff --git a/examples/generic/event_triggered/tusd/converter.go b/examples/generic/event_triggered/tusd/converter.go index 167adc00..43fe0bf2 100644 --- a/examples/generic/event_triggered/tusd/converter.go +++ b/examples/generic/event_triggered/tusd/converter.go @@ -24,8 +24,8 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) // Converter converts a raw event log into its corresponding entity diff --git a/examples/generic/event_triggered/tusd/converter_test.go b/examples/generic/event_triggered/tusd/converter_test.go index cf6d4c34..c2527022 100644 --- a/examples/generic/event_triggered/tusd/converter_test.go +++ b/examples/generic/event_triggered/tusd/converter_test.go @@ -25,8 +25,8 @@ import ( "github.com/vulcanize/vulcanizedb/examples/generic/event_triggered" "github.com/vulcanize/vulcanizedb/examples/generic/event_triggered/tusd" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) var expectedBurnModel = event_triggered.BurnModel{ diff --git a/examples/generic/event_triggered/tusd/integration_test.go b/examples/generic/event_triggered/tusd/integration_test.go index 915fe402..2b4b33b2 100644 --- a/examples/generic/event_triggered/tusd/integration_test.go +++ b/examples/generic/event_triggered/tusd/integration_test.go @@ -26,7 +26,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var burnLog = core.Log{ diff --git a/examples/generic/event_triggered/tusd/repository_test.go b/examples/generic/event_triggered/tusd/repository_test.go index 4b7b1bb1..afa66cfb 100644 --- a/examples/generic/event_triggered/tusd/repository_test.go +++ b/examples/generic/event_triggered/tusd/repository_test.go @@ -31,7 +31,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) var burnEntity = &tusd.BurnEntity{ diff --git a/examples/generic/event_triggered/tusd/transformer.go b/examples/generic/event_triggered/tusd/transformer.go index 7e8ca95e..233f0372 100644 --- a/examples/generic/event_triggered/tusd/transformer.go +++ b/examples/generic/event_triggered/tusd/transformer.go @@ -27,7 +27,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) type GenericTransformer struct { diff --git a/examples/generic/event_triggered/tusd/transformer_test.go b/examples/generic/event_triggered/tusd/transformer_test.go index 18c4cf16..34b9b408 100644 --- a/examples/generic/event_triggered/tusd/transformer_test.go +++ b/examples/generic/event_triggered/tusd/transformer_test.go @@ -23,7 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/examples/generic/event_triggered/tusd" "github.com/vulcanize/vulcanizedb/examples/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var blockID1 = int64(5428074) diff --git a/examples/generic/every_block/getter_test.go b/examples/generic/every_block/getter_test.go index 1b9f50be..cfc841af 100644 --- a/examples/generic/every_block/getter_test.go +++ b/examples/generic/every_block/getter_test.go @@ -31,7 +31,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var _ = Describe("every_block Getter", func() { diff --git a/examples/mocks/fetcher.go b/examples/mocks/fetcher.go index e343582c..c72622e6 100644 --- a/examples/mocks/fetcher.go +++ b/examples/mocks/fetcher.go @@ -111,14 +111,14 @@ func (f *Fetcher) FetchAddress(method, contractAbi, contractAddress string, bloc f.ContractAddress = contractAddress f.FetchedBlocks = append(f.FetchedBlocks, blockNumber) - adr := common.StringToAddress("test_address") + adr := common.HexToAddress("test_address") if method == "owner" { f.owner = adr return f.owner, nil } - return common.StringToAddress(""), errors.New("invalid method argument") + return common.HexToAddress(""), errors.New("invalid method argument") } func (f *Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) { @@ -148,15 +148,15 @@ func (f *Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNu f.FetchedBlocks = append(f.FetchedBlocks, blockNumber) if method == "name" { - f.hashName = common.StringToHash("test_name") + f.hashName = common.HexToHash("test_name") return f.hashName, nil } if method == "symbol" { - f.hashSymbol = common.StringToHash("test_symbol") + f.hashSymbol = common.HexToHash("test_symbol") return f.hashSymbol, nil } - return common.StringToHash(""), errors.New("invalid method argument") + return common.HexToHash(""), errors.New("invalid method argument") } diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index d6753f5d..9b182f02 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -16,13 +16,19 @@ package core -import "math/big" +import ( + "math/big" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core/types" +) type BlockChain interface { ContractDataFetcher GetBlockByNumber(blockNumber int64) (Block, error) GetHeaderByNumber(blockNumber int64) (Header, error) GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error) + GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) LastBlock() *big.Int Node() Node } diff --git a/pkg/crypto/parser.go b/pkg/crypto/parser.go index 0359fe29..91af01a8 100644 --- a/pkg/crypto/parser.go +++ b/pkg/crypto/parser.go @@ -18,7 +18,7 @@ package crypto import ( "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/discv5" ) type PublicKeyParser interface { @@ -32,6 +32,6 @@ func (EthPublicKeyParser) ParsePublicKey(privateKey string) (string, error) { if err != nil { return "", err } - pubKey := discover.PubkeyID(&np.PublicKey) + pubKey := discv5.PubkeyID(&np.PublicKey) return pubKey.String(), nil } diff --git a/pkg/datastore/ethereum/level/database.go b/pkg/datastore/ethereum/level/database.go index 6a5f023e..6deb7125 100644 --- a/pkg/datastore/ethereum/level/database.go +++ b/pkg/datastore/ethereum/level/database.go @@ -52,5 +52,5 @@ func (l LevelDatabase) GetBlockReceipts(blockHash []byte, blockNumber int64) typ func (l LevelDatabase) GetHeadBlockNumber() int64 { h := l.reader.GetHeadBlockHash() n := l.reader.GetBlockNumber(h) - return int64(n) + return int64(*n) } diff --git a/pkg/datastore/ethereum/level/database_reader.go b/pkg/datastore/ethereum/level/database_reader.go index f9281b86..66f49c8d 100644 --- a/pkg/datastore/ethereum/level/database_reader.go +++ b/pkg/datastore/ethereum/level/database_reader.go @@ -18,42 +18,42 @@ package level import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" ) type Reader interface { GetBlock(hash common.Hash, number uint64) *types.Block - GetBlockNumber(hash common.Hash) uint64 + GetBlockNumber(hash common.Hash) *uint64 GetBlockReceipts(hash common.Hash, number uint64) types.Receipts GetCanonicalHash(number uint64) common.Hash GetHeadBlockHash() common.Hash } type LevelDatabaseReader struct { - reader core.DatabaseReader + reader rawdb.DatabaseReader } -func NewLevelDatabaseReader(reader core.DatabaseReader) *LevelDatabaseReader { +func NewLevelDatabaseReader(reader rawdb.DatabaseReader) *LevelDatabaseReader { return &LevelDatabaseReader{reader: reader} } func (ldbr *LevelDatabaseReader) GetBlock(hash common.Hash, number uint64) *types.Block { - return core.GetBlock(ldbr.reader, hash, number) + return rawdb.ReadBlock(ldbr.reader, hash, number) } -func (ldbr *LevelDatabaseReader) GetBlockNumber(hash common.Hash) uint64 { - return core.GetBlockNumber(ldbr.reader, hash) +func (ldbr *LevelDatabaseReader) GetBlockNumber(hash common.Hash) *uint64 { + return rawdb.ReadHeaderNumber(ldbr.reader, hash) } func (ldbr *LevelDatabaseReader) GetBlockReceipts(hash common.Hash, number uint64) types.Receipts { - return core.GetBlockReceipts(ldbr.reader, hash, number) + return rawdb.ReadReceipts(ldbr.reader, hash, number) } func (ldbr *LevelDatabaseReader) GetCanonicalHash(number uint64) common.Hash { - return core.GetCanonicalHash(ldbr.reader, number) + return rawdb.ReadCanonicalHash(ldbr.reader, number) } func (ldbr *LevelDatabaseReader) GetHeadBlockHash() common.Hash { - return core.GetHeadBlockHash(ldbr.reader) + return rawdb.ReadHeadBlockHash(ldbr.reader) } diff --git a/pkg/fakes/mock_blockchain.go b/pkg/fakes/mock_blockchain.go index cd919011..2951ad7e 100644 --- a/pkg/fakes/mock_blockchain.go +++ b/pkg/fakes/mock_blockchain.go @@ -19,6 +19,8 @@ package fakes import ( "math/big" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -33,6 +35,9 @@ type MockBlockChain struct { fetchContractDataPassedResult interface{} fetchContractDataPassedBlockNumber int64 getBlockByNumberErr error + logQuery ethereum.FilterQuery + logQueryErr error + logQueryReturnLogs []types.Log lastBlock *big.Int node core.Node } @@ -55,6 +60,14 @@ func (blockChain *MockBlockChain) SetGetBlockByNumberErr(err error) { blockChain.getBlockByNumberErr = err } +func (chain *MockBlockChain) SetGetEthLogsWithCustomQueryErr(err error) { + chain.logQueryErr = err +} + +func (chain *MockBlockChain) SetGetEthLogsWithCustomQueryReturnLogs(logs []types.Log) { + chain.logQueryReturnLogs = logs +} + func (blockChain *MockBlockChain) GetHeaderByNumber(blockNumber int64) (core.Header, error) { return core.Header{BlockNumber: blockNumber}, nil } @@ -69,31 +82,42 @@ func (blockChain *MockBlockChain) FetchContractData(abiJSON string, address stri return blockChain.fetchContractDataErr } -func (blockChain *MockBlockChain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { - return []byte{}, nil +func (chain *MockBlockChain) GetBlockByNumber(blockNumber int64) (core.Block, error) { + return core.Block{Number: blockNumber}, chain.getBlockByNumberErr } -func (blockChain *MockBlockChain) LastBlock() *big.Int { - return blockChain.lastBlock +func (blockChain *MockBlockChain) GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error) { + blockChain.logQuery = query + return blockChain.logQueryReturnLogs, blockChain.logQueryErr } -func (blockChain *MockBlockChain) GetLogs(contract core.Contract, startingBlock *big.Int, endingBlock *big.Int) ([]core.Log, error) { +func (chain *MockBlockChain) GetLogs(contract core.Contract, startingBlockNumber, endingBlockNumber *big.Int) ([]core.Log, error) { return []core.Log{}, nil } -func (blockChain *MockBlockChain) Node() core.Node { - return blockChain.node +func (chain *MockBlockChain) CallContract(contractHash string, input []byte, blockNumber *big.Int) ([]byte, error) { + return []byte{}, nil } -func (blockChain *MockBlockChain) GetBlockByNumber(blockNumber int64) (core.Block, error) { - return core.Block{Number: blockNumber}, blockChain.getBlockByNumberErr +func (chain *MockBlockChain) LastBlock() *big.Int { + return chain.lastBlock } -// TODO: handle methodArg being nil (can't match nil to nil in Gomega) -func (blockChain *MockBlockChain) AssertFetchContractDataCalledWith(abiJSON string, address string, method string, methodArgs []interface{}, result interface{}, blockNumber int64) { - Expect(blockChain.fetchContractDataPassedAbi).To(Equal(abiJSON)) - Expect(blockChain.fetchContractDataPassedAddress).To(Equal(address)) - Expect(blockChain.fetchContractDataPassedMethod).To(Equal(method)) - Expect(blockChain.fetchContractDataPassedResult).To(Equal(result)) - Expect(blockChain.fetchContractDataPassedBlockNumber).To(Equal(blockNumber)) +func (chain *MockBlockChain) Node() core.Node { + return chain.node +} + +func (chain *MockBlockChain) AssertFetchContractDataCalledWith(abiJSON string, address string, method string, methodArgs []interface{}, result interface{}, blockNumber int64) { + Expect(chain.fetchContractDataPassedAbi).To(Equal(abiJSON)) + Expect(chain.fetchContractDataPassedAddress).To(Equal(address)) + Expect(chain.fetchContractDataPassedMethod).To(Equal(method)) + if methodArgs != nil { + Expect(chain.fetchContractDataPassedMethodArgs).To(Equal(methodArgs)) + } + Expect(chain.fetchContractDataPassedResult).To(BeAssignableToTypeOf(result)) + Expect(chain.fetchContractDataPassedBlockNumber).To(Equal(blockNumber)) +} + +func (blockChain *MockBlockChain) AssertGetEthLogsWithCustomQueryCalledWith(query ethereum.FilterQuery) { + Expect(blockChain.logQuery).To(Equal(query)) } diff --git a/pkg/fakes/mock_level_database_reader.go b/pkg/fakes/mock_level_database_reader.go index d0efd9bb..261d6be0 100644 --- a/pkg/fakes/mock_level_database_reader.go +++ b/pkg/fakes/mock_level_database_reader.go @@ -98,10 +98,10 @@ func (mldr *MockLevelDatabaseReader) GetBlockReceipts(hash common.Hash, number u return mldr.returnReceipts } -func (mldr *MockLevelDatabaseReader) GetBlockNumber(hash common.Hash) uint64 { +func (mldr *MockLevelDatabaseReader) GetBlockNumber(hash common.Hash) *uint64 { mldr.getBlockNumberCalled = true mldr.getBlockNumberPassedHash = hash - return mldr.returnBlockNumber + return &mldr.returnBlockNumber } func (mldr *MockLevelDatabaseReader) GetCanonicalHash(number uint64) common.Hash { diff --git a/pkg/geth/converters/common/block_converter_test.go b/pkg/geth/converters/common/block_converter_test.go index e3f04398..0df05be4 100644 --- a/pkg/geth/converters/common/block_converter_test.go +++ b/pkg/geth/converters/common/block_converter_test.go @@ -234,7 +234,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() { CumulativeGasUsed: uint64(7996119), GasUsed: uint64(21000), Logs: []*types.Log{}, - Status: uint(1), + Status: uint64(1), TxHash: gethTransaction.Hash(), } diff --git a/pkg/geth/converters/common/receipt_converter_test.go b/pkg/geth/converters/common/receipt_converter_test.go index ed120644..b43b6152 100644 --- a/pkg/geth/converters/common/receipt_converter_test.go +++ b/pkg/geth/converters/common/receipt_converter_test.go @@ -70,7 +70,7 @@ var _ = Describe("Conversion of GethReceipt to core.Receipt", func() { CumulativeGasUsed: uint64(7996119), GasUsed: uint64(21000), Logs: []*types.Log{}, - Status: uint(1), + Status: uint64(1), TxHash: common.HexToHash("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223"), } diff --git a/pkg/omni/converter/converter.go b/pkg/omni/full/converter/converter.go similarity index 81% rename from pkg/omni/converter/converter.go rename to pkg/omni/full/converter/converter.go index 70464337..69b0b629 100644 --- a/pkg/omni/converter/converter.go +++ b/pkg/omni/full/converter/converter.go @@ -18,6 +18,7 @@ package converter import ( "errors" + "fmt" "math/big" "strconv" @@ -25,9 +26,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Converter is used to convert watched event logs to @@ -38,27 +39,22 @@ type Converter interface { } type converter struct { - contractInfo *contract.Contract + ContractInfo *contract.Contract } func NewConverter(info *contract.Contract) *converter { - return &converter{ - contractInfo: info, + ContractInfo: info, } } func (c *converter) Update(info *contract.Contract) { - c.contractInfo = info -} - -func (c *converter) CheckInfo() *contract.Contract { - return c.contractInfo + c.ContractInfo = info } // Convert the given watched event log into a types.Log for the given event func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { - contract := bind.NewBoundContract(common.HexToAddress(c.contractInfo.Address), c.contractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) for _, field := range event.Fields { @@ -85,7 +81,7 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( var a common.Address a = input.(common.Address) strValues[fieldName] = a.String() - c.contractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses + c.ContractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses case common.Hash: var h common.Hash h = input.(common.Hash) @@ -95,12 +91,12 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( case bool: strValues[fieldName] = strconv.FormatBool(input.(bool)) default: - return nil, errors.New("error: unhandled abi type") + return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input)) } } // Only hold onto logs that pass our address filter, if any - if c.contractInfo.PassesEventFilter(strValues) { + if c.ContractInfo.PassesEventFilter(strValues) { eventLog := &types.Log{ Event: event, Id: watchedEvent.LogID, @@ -109,7 +105,7 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( Tx: watchedEvent.TxHash, } - return eventLog, err + return eventLog, nil } return nil, nil diff --git a/pkg/omni/converter/converter_suite_test.go b/pkg/omni/full/converter/converter_suite_test.go similarity index 95% rename from pkg/omni/converter/converter_suite_test.go rename to pkg/omni/full/converter/converter_suite_test.go index 0b53b314..93fd8aef 100644 --- a/pkg/omni/converter/converter_suite_test.go +++ b/pkg/omni/full/converter/converter_suite_test.go @@ -27,7 +27,7 @@ import ( func TestConverter(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Converter Suite Test") + RunSpecs(t, "Full Converter Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/omni/converter/converter_test.go b/pkg/omni/full/converter/converter_test.go similarity index 63% rename from pkg/omni/converter/converter_test.go rename to pkg/omni/full/converter/converter_test.go index 9ef7b3b0..7ce0c00b 100644 --- a/pkg/omni/converter/converter_test.go +++ b/pkg/omni/full/converter/converter_test.go @@ -21,28 +21,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) -var mockEvent = core.WatchedEvent{ - LogID: 1, - Name: constants.TransferEvent.String(), - BlockNumber: 5488076, - Address: constants.TusdContractAddress, - TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", - Index: 110, - Topic0: constants.TransferEvent.Signature(), - Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21", - Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", - Topic3: "", - Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", -} - var _ = Describe("Converter", func() { var info *contract.Contract var wantedEvents = []string{"Transfer"} @@ -55,13 +39,11 @@ var _ = Describe("Converter", func() { Describe("Update", func() { It("Updates contract info held by the converter", func() { c := converter.NewConverter(info) - i := c.CheckInfo() - Expect(i).To(Equal(info)) + Expect(c.ContractInfo).To(Equal(info)) info := test_helpers.SetupTusdContract([]string{}, []string{}) c.Update(info) - i = c.CheckInfo() - Expect(i).To(Equal(info)) + Expect(c.ContractInfo).To(Equal(info)) }) }) @@ -76,7 +58,7 @@ var _ = Describe("Converter", func() { Expect(err).ToNot(HaveOccurred()) c := converter.NewConverter(info) - log, err := c.Convert(mockEvent, event) + log, err := c.Convert(test_helpers.MockTranferEvent, event) Expect(err).ToNot(HaveOccurred()) from := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21") @@ -93,7 +75,7 @@ var _ = Describe("Converter", func() { It("Fails with an empty contract", func() { event := info.Events["Transfer"] c := converter.NewConverter(&contract.Contract{}) - _, err = c.Convert(mockEvent, event) + _, err = c.Convert(test_helpers.MockTranferEvent, event) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/omni/repository/event_repository.go b/pkg/omni/full/repository/event_repository.go similarity index 79% rename from pkg/omni/repository/event_repository.go rename to pkg/omni/full/repository/event_repository.go index e38672db..e0998308 100644 --- a/pkg/omni/repository/event_repository.go +++ b/pkg/omni/full/repository/event_repository.go @@ -22,31 +22,31 @@ import ( "strings" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) -// Event datastore is used to persist event data into custom tables -type EventDatastore interface { +// Event repository is used to persist event data into custom tables +type EventRepository interface { PersistLog(event types.Log, contractAddr, contractName string) error CreateEventTable(contractName string, event types.Log) (bool, error) CreateContractSchema(contractName string) (bool, error) } -type eventDatastore struct { - *postgres.DB +type eventRepository struct { + db *postgres.DB } -func NewEventDataStore(db *postgres.DB) *eventDatastore { +func NewEventRepository(db *postgres.DB) *eventRepository { - return &eventDatastore{ - DB: db, + return &eventRepository{ + db: db, } } // Creates a schema for the contract if needed // Creates table for the watched contract event if needed // Persists converted event log data into this custom table -func (d *eventDatastore) PersistLog(event types.Log, contractAddr, contractName string) error { +func (d *eventRepository) PersistLog(event types.Log, contractAddr, contractName string) error { _, err := d.CreateContractSchema(contractAddr) if err != nil { return err @@ -61,7 +61,7 @@ func (d *eventDatastore) PersistLog(event types.Log, contractAddr, contractName } // Creates a custom postgres command to persist logs for the given event -func (d *eventDatastore) persistLog(event types.Log, contractAddr, contractName string) error { +func (d *eventRepository) persistLog(event types.Log, contractAddr, contractName string) error { // Begin postgres string pgStr := fmt.Sprintf("INSERT INTO c%s.%s_event ", strings.ToLower(contractAddr), strings.ToLower(event.Name)) pgStr = pgStr + "(vulcanize_log_id, token_name, block, tx" @@ -91,7 +91,7 @@ func (d *eventDatastore) persistLog(event types.Log, contractAddr, contractName } pgStr = pgStr + ") ON CONFLICT (vulcanize_log_id) DO NOTHING" - _, err := d.DB.Exec(pgStr, data...) + _, err := d.db.Exec(pgStr, data...) if err != nil { return err } @@ -100,7 +100,7 @@ func (d *eventDatastore) persistLog(event types.Log, contractAddr, contractName } // Checks for event table and creates it if it does not already exist -func (d *eventDatastore) CreateEventTable(contractAddr string, event types.Log) (bool, error) { +func (d *eventRepository) CreateEventTable(contractAddr string, event types.Log) (bool, error) { tableExists, err := d.checkForTable(contractAddr, event.Name) if err != nil { return false, err @@ -117,7 +117,7 @@ func (d *eventDatastore) CreateEventTable(contractAddr string, event types.Log) } // Creates a table for the given contract and event -func (d *eventDatastore) newEventTable(contractAddr string, event types.Log) error { +func (d *eventRepository) newEventTable(contractAddr string, event types.Log) error { // Begin pg string pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS c%s.%s_event ", strings.ToLower(contractAddr), strings.ToLower(event.Name)) pgStr = pgStr + "(id SERIAL, vulcanize_log_id INTEGER NOT NULL UNIQUE, token_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL, tx CHARACTER VARYING(66) NOT NULL," @@ -128,23 +128,23 @@ func (d *eventDatastore) newEventTable(contractAddr string, event types.Log) err } pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)" - _, err := d.DB.Exec(pgStr) + _, err := d.db.Exec(pgStr) return err } // Checks if a table already exists for the given contract and event -func (d *eventDatastore) checkForTable(contractAddr string, eventName string) (bool, error) { +func (d *eventRepository) checkForTable(contractAddr string, eventName string) (bool, error) { pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'c%s' AND table_name = '%s_event')", strings.ToLower(contractAddr), strings.ToLower(eventName)) var exists bool - err := d.DB.Get(&exists, pgStr) + err := d.db.Get(&exists, pgStr) return exists, err } // Checks for contract schema and creates it if it does not already exist -func (d *eventDatastore) CreateContractSchema(contractAddr string) (bool, error) { +func (d *eventRepository) CreateContractSchema(contractAddr string) (bool, error) { if contractAddr == "" { return false, errors.New("error: no contract address specified") } @@ -165,18 +165,18 @@ func (d *eventDatastore) CreateContractSchema(contractAddr string) (bool, error) } // Creates a schema for the given contract -func (d *eventDatastore) newContractSchema(contractAddr string) error { - _, err := d.DB.Exec("CREATE SCHEMA IF NOT EXISTS c" + strings.ToLower(contractAddr)) +func (d *eventRepository) newContractSchema(contractAddr string) error { + _, err := d.db.Exec("CREATE SCHEMA IF NOT EXISTS c" + strings.ToLower(contractAddr)) return err } // Checks if a schema already exists for the given contract -func (d *eventDatastore) checkForSchema(contractAddr string) (bool, error) { +func (d *eventRepository) checkForSchema(contractAddr string) (bool, error) { pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'c%s')", strings.ToLower(contractAddr)) var exists bool - err := d.DB.QueryRow(pgStr).Scan(&exists) + err := d.db.QueryRow(pgStr).Scan(&exists) return exists, err } diff --git a/pkg/omni/repository/event_repository_test.go b/pkg/omni/full/repository/event_repository_test.go similarity index 92% rename from pkg/omni/repository/event_repository_test.go rename to pkg/omni/full/repository/event_repository_test.go index 16d4829a..bcaa3704 100644 --- a/pkg/omni/repository/event_repository_test.go +++ b/pkg/omni/full/repository/event_repository_test.go @@ -24,12 +24,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var mockEvent = core.WatchedEvent{ @@ -47,7 +47,7 @@ var mockEvent = core.WatchedEvent{ var _ = Describe("Repository", func() { var db *postgres.DB - var dataStore repository.EventDatastore + var dataStore repository.EventRepository var err error var log *types.Log var con *contract.Contract @@ -67,7 +67,7 @@ var _ = Describe("Repository", func() { log, err = c.Convert(mockEvent, event) Expect(err).ToNot(HaveOccurred()) - dataStore = repository.NewEventDataStore(db) + dataStore = repository.NewEventRepository(db) }) AfterEach(func() { diff --git a/pkg/omni/repository/repository_suite_test.go b/pkg/omni/full/repository/repository_suite_test.go similarity index 95% rename from pkg/omni/repository/repository_suite_test.go rename to pkg/omni/full/repository/repository_suite_test.go index dfc72792..899f5b04 100644 --- a/pkg/omni/repository/repository_suite_test.go +++ b/pkg/omni/full/repository/repository_suite_test.go @@ -27,7 +27,7 @@ import ( func TestRepository(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Repository Suite Test") + RunSpecs(t, "Full Repository Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/omni/retriever/address_retriever.go b/pkg/omni/full/retriever/address_retriever.go similarity index 95% rename from pkg/omni/retriever/address_retriever.go rename to pkg/omni/full/retriever/address_retriever.go index d9388abb..40609279 100644 --- a/pkg/omni/retriever/address_retriever.go +++ b/pkg/omni/full/retriever/address_retriever.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" ) // Address retriever is used to retrieve the addresses associated with a contract @@ -35,13 +35,12 @@ type AddressRetriever interface { } type addressRetriever struct { - *postgres.DB + db *postgres.DB } func NewAddressRetriever(db *postgres.DB) (r *addressRetriever) { - return &addressRetriever{ - DB: db, + db: db, } } @@ -86,7 +85,7 @@ func (r *addressRetriever) retrieveTransferAddresses(con contract.Contract) ([]s if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses addrs := make([]string, 0) pgStr := fmt.Sprintf("SELECT %s_ FROM c%s.%s_event", strings.ToLower(field.Name), strings.ToLower(con.Address), strings.ToLower(event.Name)) - err := r.DB.Select(&addrs, pgStr) + err := r.db.Select(&addrs, pgStr) if err != nil { return []string{}, err } @@ -107,7 +106,7 @@ func (r *addressRetriever) retrieveTokenMintees(con contract.Contract) ([]string if field.Type.T == abi.AddressTy { // If they have address type, retrieve those addresses addrs := make([]string, 0) pgStr := fmt.Sprintf("SELECT %s_ FROM c%s.%s_event", strings.ToLower(field.Name), strings.ToLower(con.Address), strings.ToLower(event.Name)) - err := r.DB.Select(&addrs, pgStr) + err := r.db.Select(&addrs, pgStr) if err != nil { return []string{}, err } diff --git a/pkg/omni/retriever/address_retriever_test.go b/pkg/omni/full/retriever/address_retriever_test.go similarity index 87% rename from pkg/omni/retriever/address_retriever_test.go rename to pkg/omni/full/retriever/address_retriever_test.go index f95ab5fa..daa91d50 100644 --- a/pkg/omni/retriever/address_retriever_test.go +++ b/pkg/omni/full/retriever/address_retriever_test.go @@ -20,16 +20,16 @@ import ( "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) var mockEvent = core.WatchedEvent{ @@ -47,7 +47,7 @@ var mockEvent = core.WatchedEvent{ var _ = Describe("Address Retriever Test", func() { var db *postgres.DB - var dataStore repository.EventDatastore + var dataStore repository.EventRepository var err error var info *contract.Contract var vulcanizeLogId int64 @@ -68,7 +68,7 @@ var _ = Describe("Address Retriever Test", func() { log, err = c.Convert(mockEvent, event) Expect(err).ToNot(HaveOccurred()) - dataStore = repository.NewEventDataStore(db) + dataStore = repository.NewEventRepository(db) dataStore.PersistLog(*log, info.Address, info.Name) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/omni/retriever/block_retriever.go b/pkg/omni/full/retriever/block_retriever.go similarity index 97% rename from pkg/omni/retriever/block_retriever.go rename to pkg/omni/full/retriever/block_retriever.go index 2854e8ef..2eb0337d 100644 --- a/pkg/omni/retriever/block_retriever.go +++ b/pkg/omni/full/retriever/block_retriever.go @@ -28,13 +28,12 @@ type BlockRetriever interface { } type blockRetriever struct { - *postgres.DB + db *postgres.DB } func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { - return &blockRetriever{ - DB: db, + db: db, } } @@ -51,7 +50,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { var firstBlock int - err := r.DB.Get( + err := r.db.Get( &firstBlock, `SELECT number FROM blocks WHERE id = (SELECT block_id FROM receipts @@ -67,7 +66,7 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in // In which case this servers as a heuristic to find the first block by finding the first contract event log func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) { var firstBlock int - err := r.DB.Get( + err := r.db.Get( &firstBlock, "SELECT block_number FROM logs WHERE address = $1 ORDER BY block_number ASC LIMIT 1", contractAddr, @@ -79,7 +78,7 @@ func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, // Method to retrieve the most recent block in vDB func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) { var lastBlock int64 - err := r.DB.Get( + err := r.db.Get( &lastBlock, "SELECT number FROM blocks ORDER BY number DESC LIMIT 1", ) diff --git a/pkg/omni/retriever/block_retriever_test.go b/pkg/omni/full/retriever/block_retriever_test.go similarity index 97% rename from pkg/omni/retriever/block_retriever_test.go rename to pkg/omni/full/retriever/block_retriever_test.go index d77bbacc..eaef15a2 100644 --- a/pkg/omni/retriever/block_retriever_test.go +++ b/pkg/omni/full/retriever/block_retriever_test.go @@ -23,9 +23,9 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) var _ = Describe("Block Retriever", func() { diff --git a/pkg/omni/retriever/retriever_suite_test.go b/pkg/omni/full/retriever/retriever_suite_test.go similarity index 95% rename from pkg/omni/retriever/retriever_suite_test.go rename to pkg/omni/full/retriever/retriever_suite_test.go index 4f1d878d..d5815a7c 100644 --- a/pkg/omni/retriever/retriever_suite_test.go +++ b/pkg/omni/full/retriever/retriever_suite_test.go @@ -27,7 +27,7 @@ import ( func TestRetriever(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Retriever Suite Test") + RunSpecs(t, "Full Retriever Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/omni/transformer/transformer.go b/pkg/omni/full/transformer/transformer.go similarity index 87% rename from pkg/omni/transformer/transformer.go rename to pkg/omni/full/transformer/transformer.go index f9afff05..b9987fcb 100644 --- a/pkg/omni/transformer/transformer.go +++ b/pkg/omni/full/transformer/transformer.go @@ -25,34 +25,20 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" ) -// Omni transformer -// Used to extract all or a subset of event and method data -// for any contract and persist it to postgres in a manner -// that requires no prior knowledge of the contract -// other than its address and which network it is on -type Transformer interface { - SetEvents(contractAddr string, filterSet []string) - SetEventAddrs(contractAddr string, filterSet []string) - SetMethods(contractAddr string, filterSet []string) - SetMethodAddrs(contractAddr string, filterSet []string) - SetRange(contractAddr string, rng [2]int64) - Init() error - Execute() error -} - +// Requires a fully synced vDB and a running eth node (or infura) type transformer struct { // Database interfaces datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() datastore.WatchedEventRepository // Watched event log views, created by the log filters - repository.EventDatastore // Holds transformed watched event log data + repository.EventRepository // Holds transformed watched event log data // Pre-processing interfaces parser.Parser // Parses events and methods out of contract abi fetched using contract address @@ -84,7 +70,6 @@ type transformer struct { // Transformer takes in config for blockchain, database, and network id func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transformer { - return &transformer{ Poller: poller.NewPoller(BC, DB), Parser: parser.NewParser(network), @@ -93,7 +78,7 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf Contracts: map[string]*contract.Contract{}, WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, FilterRepository: repositories.FilterRepository{DB: DB}, - EventDatastore: repository.NewEventDataStore(DB), + EventRepository: repository.NewEventRepository(DB), WatchedEvents: map[string][]string{}, WantedMethods: map[string][]string{}, ContractRanges: map[string][2]int64{}, @@ -107,7 +92,6 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf // Uses parser to pull event info from abi // Use this info to generate event filters func (t *transformer) Init() error { - for contractAddr, subset := range t.WatchedEvents { // Get Abi err := t.Parser.Parse(contractAddr) @@ -153,6 +137,7 @@ func (t *transformer) Init() error { // Aggregate info into contract object info := &contract.Contract{ Name: *name, + Network: t.Network, Address: contractAddr, Abi: t.Abi(), ParsedAbi: t.ParsedAbi(), @@ -196,7 +181,7 @@ func (tr transformer) Execute() error { for _, con := range tr.Contracts { // Update converter with current contract - tr.Converter.Update(con) + tr.Update(con) // Iterate through contract filters and get watched event logs for eventName, filter := range con.Filters { @@ -213,8 +198,11 @@ func (tr transformer) Execute() error { if err != nil { return err } + if log == nil { + break + } - // And immediately persist converted logs in repo + // If log is not empty, immediately persist in repo // Run this in seperate goroutine? err = tr.PersistLog(*log, con.Address, con.Name) if err != nil { diff --git a/pkg/omni/transformer/transformer_suite_test.go b/pkg/omni/full/transformer/transformer_suite_test.go similarity index 95% rename from pkg/omni/transformer/transformer_suite_test.go rename to pkg/omni/full/transformer/transformer_suite_test.go index 4569b47a..1f362f2b 100644 --- a/pkg/omni/transformer/transformer_suite_test.go +++ b/pkg/omni/full/transformer/transformer_suite_test.go @@ -27,7 +27,7 @@ import ( func TestTransformer(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Transformer Suite Test") + RunSpecs(t, "Full Transformer Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/omni/transformer/transformer_test.go b/pkg/omni/full/transformer/transformer_test.go similarity index 72% rename from pkg/omni/transformer/transformer_test.go rename to pkg/omni/full/transformer/transformer_test.go index 7f3d01eb..83a72073 100644 --- a/pkg/omni/transformer/transformer_test.go +++ b/pkg/omni/full/transformer/transformer_test.go @@ -27,61 +27,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/transformer" + "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) -var block1 = core.Block{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", - Number: 6194633, - Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", - Receipt: core.Receipt{ - TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", - ContractAddress: "", - Logs: []core.Log{{ - BlockNumber: 6194633, - TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", - Address: constants.TusdContractAddress, - Topics: core.Topics{ - constants.TransferEvent.Signature(), - "0x000000000000000000000000000000000000000000000000000000000000af21", - "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", - "", - }, - Index: 1, - Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", - }}, - }, - }}, -} - -var block2 = core.Block{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ooo", - Number: 6194634, - Transactions: []core.Transaction{{ - Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", - Receipt: core.Receipt{ - TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", - ContractAddress: "", - Logs: []core.Log{{ - BlockNumber: 6194634, - TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", - Address: constants.TusdContractAddress, - Topics: core.Topics{ - constants.TransferEvent.Signature(), - "0x000000000000000000000000000000000000000000000000000000000000af21", - "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", - "", - }, - Index: 1, - Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", - }}, - }, - }}, -} - var _ = Describe("Transformer", func() { var db *postgres.DB var err error @@ -145,8 +95,8 @@ var _ = Describe("Transformer", func() { Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) + blockRepository.CreateOrUpdateBlock(test_helpers.TransferBlock1) + blockRepository.CreateOrUpdateBlock(test_helpers.TransferBlock2) t := transformer.NewTransformer("", blockChain, db) t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) err = t.Init() @@ -169,9 +119,9 @@ var _ = Describe("Transformer", func() { Expect(err).To(HaveOccurred()) }) - It("Does nothing if watched events are nil", func() { - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) + It("Does nothing if watched events are unset", func() { + blockRepository.CreateOrUpdateBlock(test_helpers.TransferBlock1) + blockRepository.CreateOrUpdateBlock(test_helpers.TransferBlock2) t := transformer.NewTransformer("", blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -183,8 +133,8 @@ var _ = Describe("Transformer", func() { Describe("Execute", func() { BeforeEach(func() { - blockRepository.CreateOrUpdateBlock(block1) - blockRepository.CreateOrUpdateBlock(block2) + blockRepository.CreateOrUpdateBlock(test_helpers.TransferBlock1) + blockRepository.CreateOrUpdateBlock(test_helpers.TransferBlock2) }) It("Transforms watched contract data into custom repositories", func() { diff --git a/pkg/omni/light/converter/converter.go b/pkg/omni/light/converter/converter.go new file mode 100644 index 00000000..63348445 --- /dev/null +++ b/pkg/omni/light/converter/converter.go @@ -0,0 +1,50 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package converter + +import ( + "errors" + + geth "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" +) + +type Converter interface { + Convert(log geth.Log, event types.Event) (*types.Log, error) + Update(info *contract.Contract) +} + +type converter struct { + ContractInfo *contract.Contract +} + +func NewConverter(info *contract.Contract) *converter { + return &converter{ + ContractInfo: info, + } +} + +func (c *converter) Update(info *contract.Contract) { + c.ContractInfo = info +} + +// Convert the given watched event log into a types.Log for the given event +func (c *converter) Convert(log geth.Log, event types.Event) (*types.Log, error) { + return nil, errors.New("implement me") +} diff --git a/pkg/omni/light/converter/converter_suite_test.go b/pkg/omni/light/converter/converter_suite_test.go new file mode 100644 index 00000000..d69367df --- /dev/null +++ b/pkg/omni/light/converter/converter_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package converter_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConverter(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Light Converter Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/light/converter/converter_test.go b/pkg/omni/light/converter/converter_test.go new file mode 100644 index 00000000..a4d4e8ac --- /dev/null +++ b/pkg/omni/light/converter/converter_test.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package converter_test diff --git a/pkg/omni/light/fetcher/fetcher.go b/pkg/omni/light/fetcher/fetcher.go new file mode 100644 index 00000000..0f1e0375 --- /dev/null +++ b/pkg/omni/light/fetcher/fetcher.go @@ -0,0 +1,70 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package fetcher + +import ( + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/vulcanize/vulcanizedb/pkg/core" +) + +type LogFetcher interface { + FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) +} + +type SettableLogFetcher interface { + LogFetcher + SetBC(bc core.BlockChain) +} + +type Fetcher struct { + blockChain core.BlockChain +} + +func (fetcher *Fetcher) SetBC(bc core.BlockChain) { + fetcher.blockChain = bc +} + +func NewFetcher(blockchain core.BlockChain) Fetcher { + return Fetcher{ + blockChain: blockchain, + } +} + +func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) { + addresses := hexStringsToAddresses(contractAddresses) + blockHash := common.HexToHash(header.Hash) + query := ethereum.FilterQuery{ + BlockHash: &blockHash, + Addresses: addresses, + Topics: topics, + } + + return fetcher.blockChain.GetEthLogsWithCustomQuery(query) +} + +func hexStringsToAddresses(hexStrings []string) []common.Address { + var addresses []common.Address + for _, hexString := range hexStrings { + address := common.HexToAddress(hexString) + addresses = append(addresses, address) + } + + return addresses +} diff --git a/pkg/omni/light/fetcher/fetcher_suite_test.go b/pkg/omni/light/fetcher/fetcher_suite_test.go new file mode 100644 index 00000000..3b60a906 --- /dev/null +++ b/pkg/omni/light/fetcher/fetcher_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package fetcher_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestFetcher(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Fetcher Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/light/fetcher/fetcher_test.go b/pkg/omni/light/fetcher/fetcher_test.go new file mode 100644 index 00000000..f729b19a --- /dev/null +++ b/pkg/omni/light/fetcher/fetcher_test.go @@ -0,0 +1,66 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package fetcher_test + +import ( + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/fakes" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/fetcher" +) + +var _ = Describe("Fetcher", func() { + Describe("FetchLogs", func() { + It("fetches logs based on the given query", func() { + blockChain := fakes.NewMockBlockChain() + fetcher := fetcher.NewFetcher(blockChain) + header := fakes.FakeHeader + + addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"} + topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}} + + _, err := fetcher.FetchLogs(addresses, topicZeros, header) + + address1 := common.HexToAddress("0xfakeAddress") + address2 := common.HexToAddress("0xanotherFakeAddress") + Expect(err).NotTo(HaveOccurred()) + + blockHash := common.HexToHash(header.Hash) + expectedQuery := ethereum.FilterQuery{ + BlockHash: &blockHash, + Addresses: []common.Address{address1, address2}, + Topics: topicZeros, + } + blockChain.AssertGetEthLogsWithCustomQueryCalledWith(expectedQuery) + }) + + It("returns an error if fetching the logs fails", func() { + blockChain := fakes.NewMockBlockChain() + blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) + fetcher := fetcher.NewFetcher(blockChain) + + _, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{}) + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) +}) diff --git a/pkg/omni/light/repository/event_repository.go b/pkg/omni/light/repository/event_repository.go new file mode 100644 index 00000000..14e6b256 --- /dev/null +++ b/pkg/omni/light/repository/event_repository.go @@ -0,0 +1,181 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository + +import ( + "errors" + "fmt" + "strings" + + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" +) + +type EventRepository interface { + PersistLog(event types.Log, contractAddr, contractName string) error + CreateEventTable(contractName string, event types.Log) (bool, error) + CreateContractSchema(contractName string) (bool, error) +} + +type eventRepository struct { + db *postgres.DB +} + +func NewEventRepository(db *postgres.DB) *eventRepository { + return &eventRepository{ + db: db, + } +} + +// Creates a schema for the contract if needed +// Creates table for the watched contract event if needed +// Persists converted event log data into this custom table +func (r *eventRepository) PersistLog(event types.Log, contractAddr, contractName string) error { + _, err := r.CreateContractSchema(contractAddr) + if err != nil { + return err + } + + _, err = r.CreateEventTable(contractAddr, event) + if err != nil { + return err + } + + return r.persistLog(event, contractAddr, contractName) +} + +// Creates a custom postgres command to persist logs for the given event +func (r *eventRepository) persistLog(event types.Log, contractAddr, contractName string) error { + // Begin postgres string + pgStr := fmt.Sprintf("INSERT INTO l%s.%s_event ", strings.ToLower(contractAddr), strings.ToLower(event.Name)) + pgStr = pgStr + "(header_id, token_name, raw_log, log_idx, tx_idx" + + // Pack the corresponding variables in a slice + var data []interface{} + data = append(data, + event.Id, + contractName, + event.Raw, + event.LogIndex, + event.TransactionIndex) + + // Iterate over name-value pairs in the log adding + // names to the string and pushing values to the slice + counter := 0 // Keep track of number of inputs + for inputName, input := range event.Values { + counter += 1 + pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(inputName)) // Add underscore after to avoid any collisions with reserved pg words + data = append(data, input) + } + + // Finish off the string and execute the command using the packed data + // For each input entry we created we add its postgres command variable to the string + pgStr = pgStr + ") VALUES ($1, $2, $3, $4, $5" + for i := 0; i < counter; i++ { + pgStr = pgStr + fmt.Sprintf(", $%d", i+6) + } + pgStr = pgStr + ")" + + _, err := r.db.Exec(pgStr, data...) + if err != nil { + return err + } + + return nil +} + +// Checks for event table and creates it if it does not already exist +func (r *eventRepository) CreateEventTable(contractAddr string, event types.Log) (bool, error) { + tableExists, err := r.checkForTable(contractAddr, event.Name) + if err != nil { + return false, err + } + + if !tableExists { + err = r.newEventTable(contractAddr, event) + if err != nil { + return false, err + } + } + + return !tableExists, nil +} + +// Creates a table for the given contract and event +func (r *eventRepository) newEventTable(contractAddr string, event types.Log) error { + // Begin pg string + pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS l%s.%s_event ", strings.ToLower(contractAddr), strings.ToLower(event.Name)) + pgStr = pgStr + "(id SERIAL, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, token_name CHARACTER VARYING(66) NOT NULL, raw_log JSONB, log_idx INTEGER NOT NULL, tx_idx INTEGER NOT NULL," + + // Iterate over event fields, using their name and pgType to grow the string + for _, field := range event.Fields { + pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(field.Name), field.PgType) + } + + pgStr = pgStr + " UNIQUE (header_id, tx_idx, log_idx))" + _, err := r.db.Exec(pgStr) + + return err +} + +// Checks if a table already exists for the given contract and event +func (r *eventRepository) checkForTable(contractAddr string, eventName string) (bool, error) { + pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'l%s' AND table_name = '%s_event')", strings.ToLower(contractAddr), strings.ToLower(eventName)) + + var exists bool + err := r.db.Get(&exists, pgStr) + + return exists, err +} + +// Checks for contract schema and creates it if it does not already exist +func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) { + if contractAddr == "" { + return false, errors.New("error: no contract address specified") + } + + schemaExists, err := r.checkForSchema(contractAddr) + if err != nil { + return false, err + } + + if !schemaExists { + err = r.newContractSchema(contractAddr) + if err != nil { + return false, err + } + } + + return !schemaExists, nil +} + +// Creates a schema for the given contract +func (r *eventRepository) newContractSchema(contractAddr string) error { + _, err := r.db.Exec("CREATE SCHEMA IF NOT EXISTS l" + strings.ToLower(contractAddr)) + + return err +} + +// Checks if a schema already exists for the given contract +func (r *eventRepository) checkForSchema(contractAddr string) (bool, error) { + pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'l%s')", strings.ToLower(contractAddr)) + + var exists bool + err := r.db.QueryRow(pgStr).Scan(&exists) + + return exists, err +} diff --git a/pkg/omni/light/repository/event_repository_test.go b/pkg/omni/light/repository/event_repository_test.go new file mode 100644 index 00000000..68165a0d --- /dev/null +++ b/pkg/omni/light/repository/event_repository_test.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository_test diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/omni/light/repository/header_repository.go new file mode 100644 index 00000000..668e64e7 --- /dev/null +++ b/pkg/omni/light/repository/header_repository.go @@ -0,0 +1,70 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository + +import ( + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type HeaderRepository interface { + MarkHeaderChecked(headerID int64, eventID string) error + MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) +} + +type headerRepository struct { + db *postgres.DB +} + +func NewHeaderRepository(db *postgres.DB) *headerRepository { + return &headerRepository{ + db: db, + } +} + +func (r *headerRepository) MarkHeaderChecked(headerID int64, eventID string) error { + _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) + VALUES ($1, $2) + ON CONFLICT (header_id) DO + UPDATE SET `+eventID+` = $2`, headerID, true) + return err +} + +func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) { + var result []core.Header + var query string + var err error + + if endingBlockNumber == -1 { + query = `SELECT headers.id, headers.block_number, headers.hash FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE) + AND headers.block_number >= $1 + AND headers.eth_node_fingerprint = $2` + err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) + } else { + query = `SELECT headers.id, headers.block_number, headers.hash FROM headers + LEFT JOIN checked_headers on headers.id = header_id + WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE) + AND headers.block_number >= $1 + AND headers.block_number <= $2 + AND headers.eth_node_fingerprint = $3` + err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) + } + + return result, err +} diff --git a/pkg/omni/light/repository/header_repository_test.go b/pkg/omni/light/repository/header_repository_test.go new file mode 100644 index 00000000..68165a0d --- /dev/null +++ b/pkg/omni/light/repository/header_repository_test.go @@ -0,0 +1,17 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository_test diff --git a/pkg/omni/light/repository/repository_suite_test.go b/pkg/omni/light/repository/repository_suite_test.go new file mode 100644 index 00000000..80215953 --- /dev/null +++ b/pkg/omni/light/repository/repository_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestRepository(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Light Repository Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/light/retriever/block_retriever.go b/pkg/omni/light/retriever/block_retriever.go new file mode 100644 index 00000000..3675b8c6 --- /dev/null +++ b/pkg/omni/light/retriever/block_retriever.go @@ -0,0 +1,60 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package retriever + +import ( + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +// Block retriever is used to retrieve the first block for a given contract and the most recent block +// It requires a vDB synced database with blocks, transactions, receipts, and logs +type BlockRetriever interface { + RetrieveFirstBlock() (int64, error) + RetrieveMostRecentBlock() (int64, error) +} + +type blockRetriever struct { + db *postgres.DB +} + +func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) { + return &blockRetriever{ + db: db, + } +} + +// Retrieve block number of earliest header in repo +func (r *blockRetriever) RetrieveFirstBlock() (int64, error) { + var firstBlock int + err := r.db.Get( + &firstBlock, + "SELECT block_number FROM headers ORDER BY block_number ASC LIMIT 1", + ) + + return int64(firstBlock), err +} + +// Retrieve block number of latest header in repo +func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) { + var lastBlock int + err := r.db.Get( + &lastBlock, + "SELECT block_number FROM headers ORDER BY block_number DESC LIMIT 1", + ) + + return int64(lastBlock), err +} diff --git a/pkg/omni/light/retriever/block_retriever_test.go b/pkg/omni/light/retriever/block_retriever_test.go new file mode 100644 index 00000000..09be7b77 --- /dev/null +++ b/pkg/omni/light/retriever/block_retriever_test.go @@ -0,0 +1,78 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package retriever_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" +) + +var _ = Describe("Block Retriever", func() { + var db *postgres.DB + var r retriever.BlockRetriever + var headerRepository repositories.HeaderRepository + + BeforeEach(func() { + db, _ = test_helpers.SetupDBandBC() + headerRepository = repositories.NewHeaderRepository(db) + r = retriever.NewBlockRetriever(db) + }) + + AfterEach(func() { + test_helpers.TearDown(db) + }) + + Describe("RetrieveFirstBlock", func() { + It("Retrieves block number of earliest header in the database", func() { + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader1) + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader2) + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader3) + + i, err := r.RetrieveFirstBlock() + Expect(err).NotTo(HaveOccurred()) + Expect(i).To(Equal(int64(6194632))) + }) + + It("Fails if no headers can be found in the database", func() { + _, err := r.RetrieveFirstBlock() + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("RetrieveMostRecentBlock", func() { + It("Retrieves the latest header's block number", func() { + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader1) + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader2) + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader3) + + i, err := r.RetrieveMostRecentBlock() + Expect(err).ToNot(HaveOccurred()) + Expect(i).To(Equal(int64(6194634))) + }) + + It("Fails if no headers can be found in the database", func() { + i, err := r.RetrieveMostRecentBlock() + Expect(err).To(HaveOccurred()) + Expect(i).To(Equal(int64(0))) + }) + }) +}) diff --git a/pkg/omni/light/retriever/retriever_suite_test.go b/pkg/omni/light/retriever/retriever_suite_test.go new file mode 100644 index 00000000..dc577626 --- /dev/null +++ b/pkg/omni/light/retriever/retriever_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package retriever_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestRetriever(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Light Retriever Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/light/transformer/transformer.go b/pkg/omni/light/transformer/transformer.go new file mode 100644 index 00000000..d6c17f63 --- /dev/null +++ b/pkg/omni/light/transformer/transformer.go @@ -0,0 +1,238 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/fetcher" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" +) + +// Requires a light synced vDB (headers) and a running eth node (or infura) +type transformer struct { + // Database interfaces + repository.EventRepository // Holds transformed watched event log data + repository.HeaderRepository // Interface for interaction with header repositories + + // Pre-processing interfaces + parser.Parser // Parses events and methods out of contract abi fetched using contract address + retriever.BlockRetriever // Retrieves first block for contract and current block height + + // Processing interfaces + fetcher.Fetcher // Fetches event logs, using header hashes + converter.Converter // Converts watched event logs into custom log + poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore + + // Ethereum network name; default "" is mainnet + Network string + + // Store contract info as mapping to contract address + Contracts map[string]*contract.Contract + + // Targeted subset of events/methods + // Stored as map sof contract address to events/method names of interest + WatchedEvents map[string][]string // Default/empty event list means all are watched + WantedMethods map[string][]string // Default/empty method list means none are polled + + // Block ranges to watch contracts + ContractRanges map[string][2]int64 + + // Lists of addresses to filter event or method data + // before persisting; if empty no filter is applied + EventAddrs map[string][]string + MethodAddrs map[string][]string +} + +// Transformer takes in config for blockchain, database, and network id +func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transformer { + + return &transformer{ + Poller: poller.NewPoller(bc, db), + Fetcher: fetcher.NewFetcher(bc), + Parser: parser.NewParser(network), + HeaderRepository: repository.NewHeaderRepository(db), + BlockRetriever: retriever.NewBlockRetriever(db), + Converter: converter.NewConverter(&contract.Contract{}), + Contracts: map[string]*contract.Contract{}, + EventRepository: repository.NewEventRepository(db), + WatchedEvents: map[string][]string{}, + WantedMethods: map[string][]string{}, + ContractRanges: map[string][2]int64{}, + EventAddrs: map[string][]string{}, + MethodAddrs: map[string][]string{}, + } +} + +// Use after creating and setting transformer +// Loops over all of the addr => filter sets +// Uses parser to pull event info from abi +// Use this info to generate event filters +func (tr *transformer) Init() error { + + for contractAddr, subset := range tr.WatchedEvents { + // Get Abi + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + + // Get first block for contract and most recent block for the chain + firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock() + if err != nil { + return err + } + lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock() + if err != nil { + return err + } + + // Set to specified range if it falls within the contract's bounds + if firstBlock < tr.ContractRanges[contractAddr][0] { + firstBlock = tr.ContractRanges[contractAddr][0] + } + if lastBlock > tr.ContractRanges[contractAddr][1] && tr.ContractRanges[contractAddr][1] > firstBlock { + lastBlock = tr.ContractRanges[contractAddr][1] + } + + // Get contract name + var name = new(string) + err = tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) + if err != nil { + return errors.New(fmt.Sprintf("unable to fetch contract name: %v\r\n", err)) + } + + // Remove any accidental duplicate inputs in filter addresses + EventAddrs := map[string]bool{} + for _, addr := range tr.EventAddrs[contractAddr] { + EventAddrs[addr] = true + } + MethodAddrs := map[string]bool{} + for _, addr := range tr.MethodAddrs[contractAddr] { + MethodAddrs[addr] = true + } + + // Aggregate info into contract object + info := &contract.Contract{ + Name: *name, + Network: tr.Network, + Address: contractAddr, + Abi: tr.Abi(), + StartingBlock: firstBlock, + LastBlock: lastBlock, + Events: tr.GetEvents(subset), + Methods: tr.GetAddrMethods(tr.WantedMethods[contractAddr]), + EventAddrs: EventAddrs, + MethodAddrs: MethodAddrs, + TknHolderAddrs: map[string]bool{}, + } + + // Store contract info for further processing + tr.Contracts[contractAddr] = info + } + + return nil +} + +func (tr *transformer) Execute() error { + if len(tr.Contracts) == 0 { + return errors.New("error: transformer has no initialized contracts to work with") + } + // Iterate through all internal contracts + for _, con := range tr.Contracts { + + // Update converter with current contract + tr.Update(con) + + for _, event := range con.Events { + topics := [][]common.Hash{{common.HexToHash(event.Sig())}} + eventId := event.Name + "_" + con.Address + missingHeaders, err := tr.MissingHeaders(con.StartingBlock, con.LastBlock, eventId) + if err != nil { + return err + } + + for _, header := range missingHeaders { + logs, err := tr.FetchLogs([]string{con.Address}, topics, header) + if err != nil { + return err + } + + if len(logs) < 1 { + err = tr.MarkHeaderChecked(header.Id, eventId) + if err != nil { + return err + } + + continue + } + + for _, l := range logs { + mapping, err := tr.Convert(l, event) + if err != nil { + return err + } + if mapping == nil { + break + } + + err = tr.PersistLog(*mapping, con.Address, con.Name) + if err != nil { + return err + } + } + } + } + } + + return nil +} + +// Used to set which contract addresses and which of their events to watch +func (tr *transformer) SetEvents(contractAddr string, filterSet []string) { + tr.WatchedEvents[contractAddr] = filterSet +} + +// Used to set subset of account addresses to watch events for +func (tr *transformer) SetEventAddrs(contractAddr string, filterSet []string) { + tr.EventAddrs[contractAddr] = filterSet +} + +// Used to set which contract addresses and which of their methods to call +func (tr *transformer) SetMethods(contractAddr string, filterSet []string) { + tr.WantedMethods[contractAddr] = filterSet +} + +// Used to set subset of account addresses to poll methods on +func (tr *transformer) SetMethodAddrs(contractAddr string, filterSet []string) { + tr.MethodAddrs[contractAddr] = filterSet +} + +// Used to set the block range to watch for a given address +func (tr *transformer) SetRange(contractAddr string, rng [2]int64) { + tr.ContractRanges[contractAddr] = rng +} diff --git a/pkg/omni/light/transformer/transformer_suite_test.go b/pkg/omni/light/transformer/transformer_suite_test.go new file mode 100644 index 00000000..52ab1197 --- /dev/null +++ b/pkg/omni/light/transformer/transformer_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestTransformer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Light Transformer Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/omni/light/transformer/transformer_test.go new file mode 100644 index 00000000..5f465b86 --- /dev/null +++ b/pkg/omni/light/transformer/transformer_test.go @@ -0,0 +1,136 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer_test + +import ( + "math/rand" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" +) + +var _ = Describe("Transformer", func() { + var db *postgres.DB + var err error + var blockChain core.BlockChain + var headerRepository repositories.HeaderRepository + rand.Seed(time.Now().UnixNano()) + + BeforeEach(func() { + db, blockChain = test_helpers.SetupDBandBC() + headerRepository = repositories.NewHeaderRepository(db) + }) + + AfterEach(func() { + test_helpers.TearDown(db) + }) + + Describe("SetEvents", func() { + It("Sets which events to watch from the given contract address", func() { + watchedEvents := []string{"Transfer", "Mint"} + t := transformer.NewTransformer("", blockChain, db) + t.SetEvents(constants.TusdContractAddress, watchedEvents) + Expect(t.WatchedEvents[constants.TusdContractAddress]).To(Equal(watchedEvents)) + }) + }) + + Describe("SetEventAddrs", func() { + It("Sets which account addresses to watch events for", func() { + eventAddrs := []string{"test1", "test2"} + t := transformer.NewTransformer("", blockChain, db) + t.SetEventAddrs(constants.TusdContractAddress, eventAddrs) + Expect(t.EventAddrs[constants.TusdContractAddress]).To(Equal(eventAddrs)) + }) + }) + + Describe("SetMethods", func() { + It("Sets which methods to poll at the given contract address", func() { + watchedMethods := []string{"balanceOf", "totalSupply"} + t := transformer.NewTransformer("", blockChain, db) + t.SetMethods(constants.TusdContractAddress, watchedMethods) + Expect(t.WantedMethods[constants.TusdContractAddress]).To(Equal(watchedMethods)) + }) + }) + + Describe("SetMethodAddrs", func() { + It("Sets which account addresses to poll methods against", func() { + methodAddrs := []string{"test1", "test2"} + t := transformer.NewTransformer("", blockChain, db) + t.SetMethodAddrs(constants.TusdContractAddress, methodAddrs) + Expect(t.MethodAddrs[constants.TusdContractAddress]).To(Equal(methodAddrs)) + }) + }) + + Describe("SetRange", func() { + It("Sets the block range that the contract should be watched within", func() { + rng := [2]int64{1, 100000} + t := transformer.NewTransformer("", blockChain, db) + t.SetRange(constants.TusdContractAddress, rng) + Expect(t.ContractRanges[constants.TusdContractAddress]).To(Equal(rng)) + }) + }) + + Describe("Init", func() { + It("Initializes transformer's contract objects", func() { + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader1) + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader3) + t := transformer.NewTransformer("", blockChain, db) + t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[constants.TusdContractAddress] + Expect(ok).To(Equal(true)) + + Expect(c.StartingBlock).To(Equal(int64(6194632))) + Expect(c.LastBlock).To(Equal(int64(6194634))) + Expect(c.Abi).To(Equal(constants.TusdAbiString)) + Expect(c.Name).To(Equal("TrueUSD")) + Expect(c.Address).To(Equal(constants.TusdContractAddress)) + }) + + It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { + t := transformer.NewTransformer("", blockChain, db) + t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + err = t.Init() + Expect(err).To(HaveOccurred()) + }) + + It("Does nothing if watched events are unset", func() { + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader1) + headerRepository.CreateOrUpdateHeader(test_helpers.MockHeader3) + t := transformer.NewTransformer("", blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + _, ok := t.Contracts[constants.TusdContractAddress] + Expect(ok).To(Equal(false)) + }) + }) + + Describe("Execute", func() { + + }) +}) diff --git a/pkg/omni/constants/constants.go b/pkg/omni/shared/constants/constants.go similarity index 99% rename from pkg/omni/constants/constants.go rename to pkg/omni/shared/constants/constants.go index c6c16d8a..9be22830 100644 --- a/pkg/omni/constants/constants.go +++ b/pkg/omni/shared/constants/constants.go @@ -19,7 +19,7 @@ package constants import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) // Event enums diff --git a/pkg/omni/contract/contract.go b/pkg/omni/shared/contract/contract.go similarity index 80% rename from pkg/omni/contract/contract.go rename to pkg/omni/shared/contract/contract.go index a3faf494..d6812048 100644 --- a/pkg/omni/contract/contract.go +++ b/pkg/omni/shared/contract/contract.go @@ -23,17 +23,19 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) +// Contract object to hold our contract data type Contract struct { - Name string - Address string - StartingBlock int64 - LastBlock int64 - Abi string - ParsedAbi abi.ABI + Name string // Name of the contract + Address string // Address of the contract + Network string // Network on which the contract is deployed; default empty "" is Ethereum mainnet + StartingBlock int64 // Starting block of the contract + LastBlock int64 // Most recent block on the network + Abi string // Abi string + ParsedAbi abi.ABI // Parsed abi Events map[string]types.Event // Map of events to their names Methods map[string]types.Method // Map of methods to their names Filters map[string]filters.LogFilter // Map of event filters to their names @@ -55,7 +57,7 @@ func (c *Contract) GenerateFilters() error { Topics: core.Topics{helpers.GenerateSignature(event.Sig())}, // move generate signatrue to pkg } } - // If no filters we generated, throw an error (no point in continuing) + // If no filters were generated, throw an error (no point in continuing with this contract) if len(c.Filters) == 0 { return errors.New("error: no filters created") } diff --git a/pkg/omni/contract/contract_suite_test.go b/pkg/omni/shared/contract/contract_suite_test.go similarity index 100% rename from pkg/omni/contract/contract_suite_test.go rename to pkg/omni/shared/contract/contract_suite_test.go diff --git a/pkg/omni/contract/contract_test.go b/pkg/omni/shared/contract/contract_test.go similarity index 98% rename from pkg/omni/contract/contract_test.go rename to pkg/omni/shared/contract/contract_test.go index e8a6f89f..dc7a5ff6 100644 --- a/pkg/omni/contract/contract_test.go +++ b/pkg/omni/shared/contract/contract_test.go @@ -20,8 +20,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) var _ = Describe("Contract", func() { diff --git a/pkg/omni/helpers/helpers.go b/pkg/omni/shared/helpers/helpers.go similarity index 100% rename from pkg/omni/helpers/helpers.go rename to pkg/omni/shared/helpers/helpers.go diff --git a/pkg/omni/helpers/mocks/parser.go b/pkg/omni/shared/helpers/mocks/parser.go similarity index 98% rename from pkg/omni/helpers/mocks/parser.go rename to pkg/omni/shared/helpers/mocks/parser.go index d6ca7ee0..6decb484 100644 --- a/pkg/omni/helpers/mocks/parser.go +++ b/pkg/omni/shared/helpers/mocks/parser.go @@ -20,7 +20,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Mock parser diff --git a/pkg/omni/helpers/test_helpers/database.go b/pkg/omni/shared/helpers/test_helpers/database.go similarity index 89% rename from pkg/omni/helpers/test_helpers/database.go rename to pkg/omni/shared/helpers/test_helpers/database.go index 6c04dd71..581b8aa9 100644 --- a/pkg/omni/helpers/test_helpers/database.go +++ b/pkg/omni/shared/helpers/test_helpers/database.go @@ -27,32 +27,15 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/filters" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/parser" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" ) -var ExpectedTransferFilter = filters.LogFilter{ - Name: "Transfer", - Address: constants.TusdContractAddress, - ToBlock: -1, - FromBlock: 6194634, - Topics: core.Topics{constants.TransferEvent.Signature()}, -} - -var ExpectedApprovalFilter = filters.LogFilter{ - Name: "Approval", - Address: constants.TusdContractAddress, - ToBlock: -1, - FromBlock: 6194634, - Topics: core.Topics{constants.ApprovalEvent.Signature()}, -} - type TransferLog struct { Id int64 `db:"id"` VulvanizeLogId int64 `db:"vulcanize_log_id"` @@ -159,6 +142,12 @@ func TearDown(db *postgres.DB) { _, err := db.Query(`DELETE FROM blocks`) Expect(err).NotTo(HaveOccurred()) + _, err = db.Query(`DELETE FROM headers`) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.Query(`DELETE FROM checked_headers`) + Expect(err).NotTo(HaveOccurred()) + _, err = db.Query(`DELETE FROM logs`) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/omni/shared/helpers/test_helpers/entities.go b/pkg/omni/shared/helpers/test_helpers/entities.go new file mode 100644 index 00000000..5c3c2eb7 --- /dev/null +++ b/pkg/omni/shared/helpers/test_helpers/entities.go @@ -0,0 +1,128 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test_helpers + +import ( + "encoding/json" + + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/filters" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" +) + +var TransferBlock1 = core.Block{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", + Number: 6194633, + Transactions: []core.Transaction{{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", + Receipt: core.Receipt{ + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", + ContractAddress: "", + Logs: []core.Log{{ + BlockNumber: 6194633, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654aaa", + Address: constants.TusdContractAddress, + Topics: core.Topics{ + constants.TransferEvent.Signature(), + "0x000000000000000000000000000000000000000000000000000000000000af21", + "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + "", + }, + Index: 1, + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", + }}, + }, + }}, +} + +var TransferBlock2 = core.Block{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ooo", + Number: 6194634, + Transactions: []core.Transaction{{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", + Receipt: core.Receipt{ + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", + ContractAddress: "", + Logs: []core.Log{{ + BlockNumber: 6194634, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee", + Address: constants.TusdContractAddress, + Topics: core.Topics{ + constants.TransferEvent.Signature(), + "0x000000000000000000000000000000000000000000000000000000000000af21", + "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + "", + }, + Index: 1, + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", + }}, + }, + }}, +} + +var ExpectedTransferFilter = filters.LogFilter{ + Name: "Transfer", + Address: constants.TusdContractAddress, + ToBlock: -1, + FromBlock: 6194634, + Topics: core.Topics{constants.TransferEvent.Signature()}, +} + +var ExpectedApprovalFilter = filters.LogFilter{ + Name: "Approval", + Address: constants.TusdContractAddress, + ToBlock: -1, + FromBlock: 6194634, + Topics: core.Topics{constants.ApprovalEvent.Signature()}, +} + +var MockTranferEvent = core.WatchedEvent{ + LogID: 1, + Name: constants.TransferEvent.String(), + BlockNumber: 5488076, + Address: constants.TusdContractAddress, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Index: 110, + Topic0: constants.TransferEvent.Signature(), + Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21", + Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + Topic3: "", + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", +} + +var rawFakeHeader, _ = json.Marshal(core.Header{}) + +var MockHeader1 = core.Header{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", + BlockNumber: 6194632, + Raw: rawFakeHeader, + Timestamp: "50000000", +} + +var MockHeader2 = core.Header{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", + BlockNumber: 6194633, + Raw: rawFakeHeader, + Timestamp: "50000015", +} + +var MockHeader3 = core.Header{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + BlockNumber: 6194634, + Raw: rawFakeHeader, + Timestamp: "50000030", +} diff --git a/pkg/omni/parser/parser.go b/pkg/omni/shared/parser/parser.go similarity index 98% rename from pkg/omni/parser/parser.go rename to pkg/omni/shared/parser/parser.go index bf49c1c5..a1cdd3ad 100644 --- a/pkg/omni/parser/parser.go +++ b/pkg/omni/shared/parser/parser.go @@ -20,7 +20,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Parser is used to fetch and parse contract ABIs diff --git a/pkg/omni/parser/parser_suite_test.go b/pkg/omni/shared/parser/parser_suite_test.go similarity index 100% rename from pkg/omni/parser/parser_suite_test.go rename to pkg/omni/shared/parser/parser_suite_test.go diff --git a/pkg/omni/parser/parser_test.go b/pkg/omni/shared/parser/parser_test.go similarity index 97% rename from pkg/omni/parser/parser_test.go rename to pkg/omni/shared/parser/parser_test.go index aba71c93..8d74593b 100644 --- a/pkg/omni/parser/parser_test.go +++ b/pkg/omni/shared/parser/parser_test.go @@ -22,9 +22,9 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/omni/parser" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" ) var _ = Describe("Parser", func() { diff --git a/pkg/omni/poller/poller.go b/pkg/omni/shared/poller/poller.go similarity index 95% rename from pkg/omni/poller/poller.go rename to pkg/omni/shared/poller/poller.go index 71ee2d49..52e0425b 100644 --- a/pkg/omni/poller/poller.go +++ b/pkg/omni/shared/poller/poller.go @@ -26,9 +26,9 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) type Poller interface { @@ -37,7 +37,7 @@ type Poller interface { } type poller struct { - repository.MethodDatastore + repository.MethodRepository bc core.BlockChain contract contract.Contract } @@ -45,8 +45,8 @@ type poller struct { func NewPoller(blockChain core.BlockChain, db *postgres.DB) *poller { return &poller{ - MethodDatastore: repository.NewMethodDatastore(db), - bc: blockChain, + MethodRepository: repository.NewMethodRepository(db), + bc: blockChain, } } diff --git a/pkg/omni/poller/poller_suite_test.go b/pkg/omni/shared/poller/poller_suite_test.go similarity index 100% rename from pkg/omni/poller/poller_suite_test.go rename to pkg/omni/shared/poller/poller_suite_test.go diff --git a/pkg/omni/poller/poller_test.go b/pkg/omni/shared/poller/poller_test.go similarity index 94% rename from pkg/omni/poller/poller_test.go rename to pkg/omni/shared/poller/poller_test.go index a71f1f27..a0ba7349 100644 --- a/pkg/omni/poller/poller_test.go +++ b/pkg/omni/shared/poller/poller_test.go @@ -24,10 +24,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/poller" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" ) var _ = Describe("Poller", func() { diff --git a/pkg/omni/repository/method_repository.go b/pkg/omni/shared/repository/method_repository.go similarity index 83% rename from pkg/omni/repository/method_repository.go rename to pkg/omni/shared/repository/method_repository.go index 24af230e..ede488d8 100644 --- a/pkg/omni/repository/method_repository.go +++ b/pkg/omni/shared/repository/method_repository.go @@ -22,27 +22,27 @@ import ( "strings" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) -type MethodDatastore interface { +type MethodRepository interface { PersistResult(method types.Result, contractAddr, contractName string) error CreateMethodTable(contractAddr string, method types.Result) (bool, error) CreateContractSchema(contractAddr string) (bool, error) } -type methodDatastore struct { +type methodRepository struct { *postgres.DB } -func NewMethodDatastore(db *postgres.DB) *methodDatastore { +func NewMethodRepository(db *postgres.DB) *methodRepository { - return &methodDatastore{ + return &methodRepository{ DB: db, } } -func (d *methodDatastore) PersistResult(method types.Result, contractAddr, contractName string) error { +func (d *methodRepository) PersistResult(method types.Result, contractAddr, contractName string) error { if len(method.Args) != len(method.Inputs) { return errors.New("error: given number of inputs does not match number of method arguments") } @@ -64,7 +64,7 @@ func (d *methodDatastore) PersistResult(method types.Result, contractAddr, contr } // Creates a custom postgres command to persist logs for the given event -func (d *methodDatastore) persistResult(method types.Result, contractAddr, contractName string) error { +func (d *methodRepository) persistResult(method types.Result, contractAddr, contractName string) error { // Begin postgres string pgStr := fmt.Sprintf("INSERT INTO c%s.%s_method ", strings.ToLower(contractAddr), strings.ToLower(method.Name)) pgStr = pgStr + "(token_name, block" @@ -103,7 +103,7 @@ func (d *methodDatastore) persistResult(method types.Result, contractAddr, contr } // Checks for event table and creates it if it does not already exist -func (d *methodDatastore) CreateMethodTable(contractAddr string, method types.Result) (bool, error) { +func (d *methodRepository) CreateMethodTable(contractAddr string, method types.Result) (bool, error) { tableExists, err := d.checkForTable(contractAddr, method.Name) if err != nil { return false, err @@ -120,7 +120,7 @@ func (d *methodDatastore) CreateMethodTable(contractAddr string, method types.Re } // Creates a table for the given contract and event -func (d *methodDatastore) newMethodTable(contractAddr string, method types.Result) error { +func (d *methodRepository) newMethodTable(contractAddr string, method types.Result) error { // Begin pg string pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS c%s.%s_method ", strings.ToLower(contractAddr), strings.ToLower(method.Name)) pgStr = pgStr + "(id SERIAL, token_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL," @@ -138,7 +138,7 @@ func (d *methodDatastore) newMethodTable(contractAddr string, method types.Resul } // Checks if a table already exists for the given contract and event -func (d *methodDatastore) checkForTable(contractAddr string, methodName string) (bool, error) { +func (d *methodRepository) checkForTable(contractAddr string, methodName string) (bool, error) { pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'c%s' AND table_name = '%s_method')", strings.ToLower(contractAddr), strings.ToLower(methodName)) var exists bool err := d.DB.Get(&exists, pgStr) @@ -147,7 +147,7 @@ func (d *methodDatastore) checkForTable(contractAddr string, methodName string) } // Checks for contract schema and creates it if it does not already exist -func (d *methodDatastore) CreateContractSchema(contractAddr string) (bool, error) { +func (d *methodRepository) CreateContractSchema(contractAddr string) (bool, error) { if contractAddr == "" { return false, errors.New("error: no contract address specified") } @@ -168,14 +168,14 @@ func (d *methodDatastore) CreateContractSchema(contractAddr string) (bool, error } // Creates a schema for the given contract -func (d *methodDatastore) newContractSchema(contractAddr string) error { +func (d *methodRepository) newContractSchema(contractAddr string) error { _, err := d.DB.Exec("CREATE SCHEMA IF NOT EXISTS c" + strings.ToLower(contractAddr)) return err } // Checks if a schema already exists for the given contract -func (d *methodDatastore) checkForSchema(contractAddr string) (bool, error) { +func (d *methodRepository) checkForSchema(contractAddr string) (bool, error) { pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'c%s')", strings.ToLower(contractAddr)) var exists bool diff --git a/pkg/omni/repository/method_repository_test.go b/pkg/omni/shared/repository/method_repository_test.go similarity index 89% rename from pkg/omni/repository/method_repository_test.go rename to pkg/omni/shared/repository/method_repository_test.go index b153ef75..80c93e53 100644 --- a/pkg/omni/repository/method_repository_test.go +++ b/pkg/omni/shared/repository/method_repository_test.go @@ -23,16 +23,16 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Repository", func() { var db *postgres.DB - var dataStore repository.MethodDatastore + var dataStore repository.MethodRepository var con *contract.Contract var err error var mockResult types.Result @@ -50,7 +50,7 @@ var _ = Describe("Repository", func() { mockResult.Inputs[0] = "0xfE9e8709d3215310075d67E3ed32A380CCf451C8" mockResult.Output = "66386309548896882859581786" db, _ = test_helpers.SetupDBandBC() - dataStore = repository.NewMethodDatastore(db) + dataStore = repository.NewMethodRepository(db) }) AfterEach(func() { diff --git a/pkg/omni/shared/repository/repository_suite_test.go b/pkg/omni/shared/repository/repository_suite_test.go new file mode 100644 index 00000000..a1bfc548 --- /dev/null +++ b/pkg/omni/shared/repository/repository_suite_test.go @@ -0,0 +1,35 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package repository_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestRepository(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Shared Repository Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/shared/transformer/interface.go b/pkg/omni/shared/transformer/interface.go new file mode 100644 index 00000000..c69f1046 --- /dev/null +++ b/pkg/omni/shared/transformer/interface.go @@ -0,0 +1,29 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer + +// Used to extract any/all events and a subset of method (state variable) +// data for any contract and persists it to custom postgres tables in vDB +type Transformer interface { + SetEvents(contractAddr string, filterSet []string) + SetEventAddrs(contractAddr string, filterSet []string) + SetMethods(contractAddr string, filterSet []string) + SetMethodAddrs(contractAddr string, filterSet []string) + SetRange(contractAddr string, rng []int64) + Init() error + Execute() error +} diff --git a/pkg/omni/shared/types/event.go b/pkg/omni/shared/types/event.go new file mode 100644 index 00000000..a2b1da84 --- /dev/null +++ b/pkg/omni/shared/types/event.go @@ -0,0 +1,98 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package types + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/i-norden/go-ethereum/core/types" +) + +type Event struct { + Name string + Anonymous bool + Fields []Field +} + +type Field struct { + abi.Argument // Name, Type, Indexed + PgType string // Holds type used when committing data held in this field to postgres +} + +// Struct to hold instance of an event log data +type Log struct { + Event + Id int64 // VulcanizeIdLog for full sync and header ID for light sync omni watcher + Values map[string]string // Map of event input names to their values + + // Used for full sync only + Block int64 + Tx string + + // Used for lightSync only + LogIndex uint + TransactionIndex uint + Raw types.Log +} + +// Unpack abi.Event into our custom Event struct +func NewEvent(e abi.Event) Event { + fields := make([]Field, len(e.Inputs)) + for i, input := range e.Inputs { + fields[i] = Field{} + fields[i].Name = input.Name + fields[i].Type = input.Type + fields[i].Indexed = input.Indexed + // Fill in pg type based on abi type + switch fields[i].Type.T { + case abi.StringTy, abi.HashTy, abi.AddressTy: + fields[i].PgType = "CHARACTER VARYING(66)" + case abi.IntTy, abi.UintTy: + fields[i].PgType = "DECIMAL" + case abi.BoolTy: + fields[i].PgType = "BOOLEAN" + case abi.BytesTy, abi.FixedBytesTy: + fields[i].PgType = "BYTEA" + case abi.ArrayTy: + fields[i].PgType = "TEXT[]" + case abi.FixedPointTy: + fields[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres? + case abi.FunctionTy: + fields[i].PgType = "TEXT" + default: + fields[i].PgType = "TEXT" + } + } + + return Event{ + Name: e.Name, + Anonymous: e.Anonymous, + Fields: fields, + } +} + +func (e Event) Sig() string { + types := make([]string, len(e.Fields)) + + for i, input := range e.Fields { + types[i] = input.Type.String() + } + + return fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ",")) +} diff --git a/pkg/omni/types/entities.go b/pkg/omni/shared/types/method.go similarity index 65% rename from pkg/omni/types/entities.go rename to pkg/omni/shared/types/method.go index c59a00ac..9c831f79 100644 --- a/pkg/omni/types/entities.go +++ b/pkg/omni/shared/types/method.go @@ -23,12 +23,6 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" ) -type Event struct { - Name string - Anonymous bool - Fields []Field -} - type Method struct { Name string Const bool @@ -36,11 +30,6 @@ type Method struct { Return []Field } -type Field struct { - abi.Argument // Name, Type, Indexed - PgType string // Holds type used when committing data held in this field to postgres -} - // Struct to hold instance of result from method call with given inputs and block type Result struct { Method @@ -50,51 +39,6 @@ type Result struct { Block int64 } -// Struct to hold instance of an event log data -type Log struct { - Event - Id int64 // VulcanizeIdLog - Values map[string]string // Map of event input names to their values - Block int64 - Tx string -} - -// Unpack abi.Event into our custom Event struct -func NewEvent(e abi.Event) Event { - fields := make([]Field, len(e.Inputs)) - for i, input := range e.Inputs { - fields[i] = Field{} - fields[i].Name = input.Name - fields[i].Type = input.Type - fields[i].Indexed = input.Indexed - // Fill in pg type based on abi type - switch fields[i].Type.T { - case abi.StringTy, abi.HashTy, abi.AddressTy: - fields[i].PgType = "CHARACTER VARYING(66)" - case abi.IntTy, abi.UintTy: - fields[i].PgType = "DECIMAL" - case abi.BoolTy: - fields[i].PgType = "BOOLEAN" - case abi.BytesTy, abi.FixedBytesTy: - fields[i].PgType = "BYTEA" - case abi.ArrayTy: - fields[i].PgType = "TEXT[]" - case abi.FixedPointTy: - fields[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres? - case abi.FunctionTy: - fields[i].PgType = "TEXT" - default: - fields[i].PgType = "TEXT" - } - } - - return Event{ - Name: e.Name, - Anonymous: e.Anonymous, - Fields: fields, - } -} - // Unpack abi.Method into our custom Method struct func NewMethod(m abi.Method) Method { inputs := make([]Field, len(m.Inputs)) @@ -157,16 +101,6 @@ func NewMethod(m abi.Method) Method { } } -func (e Event) Sig() string { - types := make([]string, len(e.Fields)) - - for i, input := range e.Fields { - types[i] = input.Type.String() - } - - return fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ",")) -} - func (m Method) Sig() string { types := make([]string, len(m.Args)) i := 0 diff --git a/vendor/github.com/ethereum/go-ethereum/interfaces.go b/vendor/github.com/ethereum/go-ethereum/interfaces.go index 1ae1eba4..26b0fcbc 100644 --- a/vendor/github.com/ethereum/go-ethereum/interfaces.go +++ b/vendor/github.com/ethereum/go-ethereum/interfaces.go @@ -131,6 +131,7 @@ type ContractCaller interface { // FilterQuery contains options for contract log filtering. type FilterQuery struct { + BlockHash *common.Hash // used by eth_getLogs, return logs only from block with this hash FromBlock *big.Int // beginning of the queried range, nil means genesis block ToBlock *big.Int // end of the range, nil means latest block Addresses []common.Address // restricts matches to events created by specific contracts @@ -144,7 +145,7 @@ type FilterQuery struct { // {} or nil matches any topic list // {{A}} matches topic A in first position // {{}, {B}} matches any topic in first position, B in second position - // {{A}}, {B}} matches topic A in first position, B in second position + // {{A}, {B}} matches topic A in first position, B in second position // {{A, B}}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position Topics [][]common.Hash }