From 8174ce4aee6172067cf0d5eb90597fef7d15ae90 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 11:19:18 -0500 Subject: [PATCH 01/10] refactor omni code; rename omniWatcher => contractWatcher; use config instead of excessive number of CLI flags --- cmd/contractWatcher.go | 123 ++++ cmd/omniWatcher.go | 121 ---- .../omni_full_transformer_test.go | 75 +-- .../omni_light_transformer_test.go | 105 ++-- pkg/config/contract.go | 177 ++++++ pkg/omni/full/converter/converter.go | 4 +- pkg/omni/full/transformer/transformer.go | 164 ++---- pkg/omni/full/transformer/transformer_test.go | 393 +++++++++---- pkg/omni/light/converter/converter.go | 8 +- .../light/repository/header_repository.go | 6 +- .../light/retriever/retriever_suite_test.go | 2 +- pkg/omni/light/transformer/transformer.go | 146 ++--- .../light/transformer/transformer_test.go | 526 ++++++++++++++---- pkg/omni/shared/constants/constants.go | 1 - pkg/omni/shared/contract/contract.go | 6 - .../shared/helpers/test_helpers/database.go | 12 +- .../helpers/test_helpers/mocks/entities.go | 77 +++ .../shared/repository/event_repository.go | 4 +- .../shared/repository/method_repository.go | 2 +- pkg/omni/shared/transformer/interface.go | 32 -- 20 files changed, 1278 insertions(+), 706 deletions(-) create mode 100644 cmd/contractWatcher.go delete mode 100644 cmd/omniWatcher.go create mode 100644 pkg/config/contract.go delete mode 100644 pkg/omni/shared/transformer/interface.go diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go new file mode 100644 index 00000000..342de614 --- /dev/null +++ b/cmd/contractWatcher.go @@ -0,0 +1,123 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" + lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" + "github.com/vulcanize/vulcanizedb/utils" +) + +// contractWatcherCmd represents the contractWatcher command +var contractWatcherCmd = &cobra.Command{ + Use: "contractWatcher", + Short: "Watches events at the provided contract address using fully synced vDB", + Long: `Uses input contract address and event filters to watch events + +Expects an ethereum node to be running +Expects an archival node synced into vulcanizeDB +Requires a .toml config file: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + + [contract] + network = "" + addresses = [ + "contractAddress1", + "contractAddress2" + ] + [contract.contractAddress1] + abi = 'ABI for contract 1' + startingBlock = 982463 + [contract.contractAddress2] + abi = 'ABI for contract 2' + events = [ + "event1", + "event2" + ] + eventArgs = [ + "arg1", + "arg2" + ] + methods = [ + "method1", + "method2" + ] + methodArgs = [ + "arg1", + "arg2" + ] + startingBlock = 4448566 + piping = true +`, + Run: func(cmd *cobra.Command, args []string) { + contractWatcher() + }, +} + +var ( + mode string +) + +func contractWatcher() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + blockChain := getBlockChain() + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + + var t st.GenericTransformer + con := config.ContractConfig{} + con.PrepConfig() + switch mode { + case "light": + t = lt.NewTransformer(con, blockChain, &db) + case "full": + t = ft.NewTransformer(con, blockChain, &db) + default: + log.Fatal("Invalid mode") + } + + err := t.Init() + if err != nil { + log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) + } + + for range ticker.C { + t.Execute() + } +} + +func init() { + rootCmd.AddCommand(contractWatcherCmd) + + contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") +} diff --git a/cmd/omniWatcher.go b/cmd/omniWatcher.go deleted file mode 100644 index a07108a9..00000000 --- a/cmd/omniWatcher.go +++ /dev/null @@ -1,121 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package cmd - -import ( - "fmt" - "log" - "time" - - "github.com/spf13/cobra" - - ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - st "github.com/vulcanize/vulcanizedb/pkg/omni/shared/transformer" - "github.com/vulcanize/vulcanizedb/utils" -) - -// omniWatcherCmd represents the omniWatcher command -var omniWatcherCmd = &cobra.Command{ - Use: "omniWatcher", - Short: "Watches events at the provided contract address using fully synced vDB", - Long: `Uses input contract address and event filters to watch events - -Expects an ethereum node to be running -Expects an archival node synced into vulcanizeDB -Requires a .toml config file: - - [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - - [client] - ipcPath = "/Users/user/Library/Ethereum/geth.ipc" -`, - Run: func(cmd *cobra.Command, args []string) { - omniWatcher() - }, -} - -var ( - network string - contractAddress string - contractAddresses []string - contractEvents []string - contractMethods []string - eventArgs []string - methodArgs []string - methodPiping bool - mode string -) - -func omniWatcher() { - if contractAddress == "" && len(contractAddresses) == 0 { - log.Fatal("Contract address required") - } - - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - blockChain := getBlockChain() - db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - - var t st.Transformer - switch mode { - case "light": - t = lt.NewTransformer(network, blockChain, &db) - case "full": - t = ft.NewTransformer(network, blockChain, &db) - default: - log.Fatal("Invalid mode") - } - - contractAddresses = append(contractAddresses, contractAddress) - for _, addr := range contractAddresses { - t.SetEvents(addr, contractEvents) - t.SetMethods(addr, contractMethods) - t.SetEventArgs(addr, eventArgs) - t.SetMethodArgs(addr, methodArgs) - t.SetPiping(addr, methodPiping) - t.SetStartingBlock(addr, startingBlockNumber) - } - - err := t.Init() - if err != nil { - log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) - } - - for range ticker.C { - t.Execute() - } -} - -func init() { - rootCmd.AddCommand(omniWatcherCmd) - - omniWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") - omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") - omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address") - omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched") - omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled") - omniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified") - omniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here") - omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) - omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists") - omniWatcherCmd.Flags().BoolVarP(&methodPiping, "piping", "p", false, "Turn on method output piping: methods listed first will be polled first and their output used as input to subsequent methods") -} diff --git a/integration_test/omni_full_transformer_test.go b/integration_test/omni_full_transformer_test.go index 5346584d..cdfb93ae 100644 --- a/integration_test/omni_full_transformer_test.go +++ b/integration_test/omni_full_transformer_test.go @@ -2,6 +2,7 @@ package integration import ( "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "math/rand" "strings" "time" @@ -41,8 +42,7 @@ var _ = Describe("Omni full transformer", func() { It("Initializes transformer's contract objects", func() { blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -64,9 +64,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -85,9 +83,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -119,9 +120,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -145,9 +149,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) @@ -161,9 +163,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -183,9 +183,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -210,9 +213,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -235,10 +241,12 @@ var _ = Describe("Omni full transformer", func() { }) It("It does not perist events if they do not pass the emitted arg filter", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -252,10 +260,15 @@ var _ = Describe("Omni full transformer", func() { }) It("If a method arg filter is applied, only those arguments are used in polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, + } + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) diff --git a/integration_test/omni_light_transformer_test.go b/integration_test/omni_light_transformer_test.go index 5634ba14..6cc8bb93 100644 --- a/integration_test/omni_light_transformer_test.go +++ b/integration_test/omni_light_transformer_test.go @@ -2,6 +2,7 @@ package integration import ( "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" @@ -39,8 +40,7 @@ var _ = Describe("Omnit light transformer", func() { It("Initializes transformer's contract objects", func() { headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -70,9 +70,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -89,9 +87,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) c, ok := t.Contracts[tusdAddr] @@ -131,9 +132,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -150,9 +154,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) }) @@ -173,9 +175,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -192,9 +192,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) c, ok := t.Contracts[ensAddr] @@ -218,9 +221,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given method using list of collected hashes", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -242,10 +248,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("It does not persist events if they do not pass the emitted arg filter", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -257,10 +265,15 @@ var _ = Describe("Omnit light transformer", func() { }) It("If a method arg filter is applied, only those arguments are used in polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, + } + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -302,11 +315,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -332,11 +341,13 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) ens, ok := t.Contracts[ensAddr] @@ -376,11 +387,13 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given methods for each contract, using list of collected values", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() diff --git a/pkg/config/contract.go b/pkg/config/contract.go new file mode 100644 index 00000000..a55933ea --- /dev/null +++ b/pkg/config/contract.go @@ -0,0 +1,177 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "strings" +) + +// Config struct for generic contract transformer +type ContractConfig struct { + // Name for the transformer + Name string + + // Ethereum network name; default "" is mainnet + Network string + + // List of contract addresses (map to ensure no duplicates) + Addresses map[string]bool + + // Map of contract address to abi + // If an address has no associated abi the parser will attempt to fetch one from etherscan + Abis map[string]string + + // Map of contract address to slice of events + // Used to set which addresses to watch + // If any events are listed in the slice only those will be watched + // Otherwise all events in the contract ABI are watched + Events map[string][]string + + // Map of contract address to slice of methods + // If any methods are listed in the slice only those will be polled + // Otherwise no methods will be polled + Methods map[string][]string + + // Map of contract address to slice of event arguments to filter for + // If arguments are provided then only events which emit those arguments are watched + // Otherwise arguments are not filtered on events + EventArgs map[string][]string + + // Map of contract address to slice of method arguments to limit polling to + // If arguments are provided then only those arguments are allowed as arguments in method polling + // Otherwise any argument of the right type seen emitted from events at that contract will be used in method polling + MethodArgs map[string][]string + + // Map of contract address to their starting block + StartingBlocks map[string]int64 + + // Map of contract address to whether or not to pipe method polling results forward into subsequent method calls + Piping map[string]bool +} + +func (oc *ContractConfig) PrepConfig() { + addrs := viper.GetStringSlice("contract.addresses") + oc.Network = viper.GetString("contract.network") + oc.Addresses = make(map[string]bool, len(addrs)) + oc.Abis = make(map[string]string, len(addrs)) + oc.Methods = make(map[string][]string, len(addrs)) + oc.EventArgs = make(map[string][]string, len(addrs)) + oc.MethodArgs = make(map[string][]string, len(addrs)) + oc.EventArgs = make(map[string][]string, len(addrs)) + oc.StartingBlocks = make(map[string]int64, len(addrs)) + oc.Piping = make(map[string]bool, len(addrs)) + // De-dupe addresses + for _, addr := range addrs { + oc.Addresses[strings.ToLower(addr)] = true + } + + // Iterate over addresses to pull out config info for each contract + for _, addr := range addrs { + transformer := viper.GetStringMap("contract." + addr) + + // Get and check abi + abi, abiOK := transformer["abi"] + if !abiOK || abi == nil { + log.Fatal(addr, "transformer config is missing `abi` value") + } + abiRef, abiOK := abi.(string) + if !abiOK { + log.Fatal(addr, "transformer `events` not of type []string") + } + oc.Abis[strings.ToLower(addr)] = abiRef + + // Get and check events + events, eventsOK := transformer["events"] + if !eventsOK || events == nil { + log.Fatal(addr, "transformer config is missing `events` value") + } + eventsRef, eventsOK := events.([]string) + if !eventsOK { + log.Fatal(addr, "transformer `events` not of type []string") + } + if eventsRef == nil { + eventsRef = []string{} + } + oc.Events[strings.ToLower(addr)] = eventsRef + + // Get and check methods + methods, methodsOK := transformer["methods"] + if !methodsOK || methods == nil { + log.Fatal(addr, "transformer config is missing `methods` value") + } + methodsRef, methodsOK := methods.([]string) + if !methodsOK { + log.Fatal(addr, "transformer `methods` not of type []string") + } + if methodsRef == nil { + methodsRef = []string{} + } + oc.Methods[strings.ToLower(addr)] = methodsRef + + // Get and check eventArgs + eventArgs, eventArgsOK := transformer["eventArgs"] + if !eventArgsOK || eventArgs == nil { + log.Fatal(addr, "transformer config is missing `eventArgs` value") + } + eventArgsRef, eventArgsOK := eventArgs.([]string) + if !eventArgsOK { + log.Fatal(addr, "transformer `eventArgs` not of type []string") + } + if eventArgsRef == nil { + eventArgsRef = []string{} + } + oc.EventArgs[strings.ToLower(addr)] = eventArgsRef + + // Get and check methodArgs + methodArgs, methodArgsOK := transformer["methodArgs"] + if !methodArgsOK || methodArgs == nil { + log.Fatal(addr, "transformer config is missing `methodArgs` value") + } + methodArgsRef, methodArgsOK := methodArgs.([]string) + if !methodArgsOK { + log.Fatal(addr, "transformer `methodArgs` not of type []string") + } + if methodArgsRef == nil { + methodArgsRef = []string{} + } + oc.MethodArgs[strings.ToLower(addr)] = methodArgsRef + + // Get and check startingBlock + start, startOK := transformer["startingBlock"] + if !startOK || start == nil { + log.Fatal(addr, "transformer config is missing `startingBlock` value") + } + startRef, startOK := start.(int64) + if !startOK { + log.Fatal(addr, "transformer `startingBlock` not of type int") + } + oc.StartingBlocks[strings.ToLower(addr)] = startRef + + // Get pipping + pipe, pipeOK := transformer["pipping"] + if !pipeOK || pipe == nil { + log.Fatal(addr, "transformer config is missing `pipping` value") + } + pipeRef, pipeOK := pipe.(bool) + if !pipeOK { + log.Fatal(addr, "transformer `piping` not of type bool") + } + oc.Piping[strings.ToLower(addr)] = pipeRef + } +} diff --git a/pkg/omni/full/converter/converter.go b/pkg/omni/full/converter/converter.go index 2810438a..1c615c33 100644 --- a/pkg/omni/full/converter/converter.go +++ b/pkg/omni/full/converter/converter.go @@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) { // Convert the given watched event log into a types.Log for the given event func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) log := helpers.ConvertToLog(watchedEvent) - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/omni/full/transformer/transformer.go b/pkg/omni/full/transformer/transformer.go index 3a37c084..6d51bdb1 100644 --- a/pkg/omni/full/transformer/transformer.go +++ b/pkg/omni/full/transformer/transformer.go @@ -18,8 +18,8 @@ package transformer import ( "errors" - "strings" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -34,7 +34,7 @@ import ( ) // Requires a fully synced vDB and a running eth node (or infura) -type Transformer struct { +type transformer struct { // Database interfaces datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() datastore.WatchedEventRepository // Watched event log views, created by the log filters @@ -48,52 +48,25 @@ type Transformer struct { converter.Converter // Converts watched event logs into custom log poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore - // Ethereum network name; default "" is mainnet - Network string + // Store contract configuration information + Config config.ContractConfig // Store contract info as mapping to contract address Contracts map[string]*contract.Contract - - // Targeted subset of events/methods - // Stored as map sof contract address to events/method names of interest - WatchedEvents map[string][]string // Default/empty event list means all are watched - WantedMethods map[string][]string // Default/empty method list means none are polled - - // Starting block for contracts - ContractStart map[string]int64 - - // Lists of addresses to filter event or method data - // before persisting; if empty no filter is applied - EventArgs map[string][]string - MethodArgs map[string][]string - - // Whether or not to create a list of emitted address or hashes for the contract in postgres - CreateAddrList map[string]bool - CreateHashList map[string]bool - - // Method piping on/off for a contract - Piping map[string]bool } // Transformer takes in config for blockchain, database, and network id -func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer { - return &Transformer{ +func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *transformer { + return &transformer{ Poller: poller.NewPoller(BC, DB, types.FullSync), - Parser: parser.NewParser(network), + Parser: parser.NewParser(con.Network), BlockRetriever: retriever.NewBlockRetriever(DB), Converter: converter.NewConverter(&contract.Contract{}), Contracts: map[string]*contract.Contract{}, WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, FilterRepository: repositories.FilterRepository{DB: DB}, EventRepository: repository.NewEventRepository(DB, types.FullSync), - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - Piping: map[string]bool{}, + Config: con, } } @@ -101,59 +74,66 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transf // Loops over all of the addr => filter sets // Uses parser to pull event info from abi // Use this info to generate event filters -func (transformer *Transformer) Init() error { - for contractAddr, subset := range transformer.WatchedEvents { - // Get Abi - err := transformer.Parser.Parse(contractAddr) - if err != nil { - return err +func (tr *transformer) Init() error { + for contractAddr := range tr.Config.Addresses { + // Configure Abi + if tr.Config.Abis[contractAddr] == "" { + // If no abi is given in the config, this method will try fetching from internal look-up table and etherscan + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + } else { + // If we have an abi from the config, load that into the parser + err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr]) + if err != nil { + return err + } } // Get first block and most recent block number in the header repo - firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr) + firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock(contractAddr) if err != nil { return err } - lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock() + lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock() if err != nil { return err } // Set to specified range if it falls within the bounds - if firstBlock < transformer.ContractStart[contractAddr] { - firstBlock = transformer.ContractStart[contractAddr] + if firstBlock < tr.Config.StartingBlocks[contractAddr] { + firstBlock = tr.Config.StartingBlocks[contractAddr] } // Get contract name if it has one var name = new(string) - transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock) + tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) // Remove any potential accidental duplicate inputs in arg filter values eventArgs := map[string]bool{} - for _, arg := range transformer.EventArgs[contractAddr] { + for _, arg := range tr.Config.EventArgs[contractAddr] { eventArgs[arg] = true } methodArgs := map[string]bool{} - for _, arg := range transformer.MethodArgs[contractAddr] { + for _, arg := range tr.Config.MethodArgs[contractAddr] { methodArgs[arg] = true } // Aggregate info into contract object info := contract.Contract{ - Name: *name, - Network: transformer.Network, - Address: contractAddr, - Abi: transformer.Parser.Abi(), - ParsedAbi: transformer.Parser.ParsedAbi(), - StartingBlock: firstBlock, - LastBlock: lastBlock, - Events: transformer.Parser.GetEvents(subset), - Methods: transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]), - FilterArgs: eventArgs, - MethodArgs: methodArgs, - CreateAddrList: transformer.CreateAddrList[contractAddr], - CreateHashList: transformer.CreateHashList[contractAddr], - Piping: transformer.Piping[contractAddr], + Name: *name, + Network: tr.Config.Network, + Address: contractAddr, + Abi: tr.Parser.Abi(), + ParsedAbi: tr.Parser.ParsedAbi(), + StartingBlock: firstBlock, + LastBlock: lastBlock, + Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), + Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + Piping: tr.Config.Piping[contractAddr], }.Init() // Use info to create filters @@ -164,14 +144,14 @@ func (transformer *Transformer) Init() error { // Iterate over filters and push them to the repo using filter repository interface for _, filter := range info.Filters { - err = transformer.FilterRepository.CreateFilter(filter) + err = tr.CreateFilter(filter) if err != nil { return err } } // Store contract info for further processing - transformer.Contracts[contractAddr] = info + tr.Contracts[contractAddr] = info } return nil @@ -182,18 +162,18 @@ func (transformer *Transformer) Init() error { // Uses converter to convert logs into custom log type // Persists converted logs into custuom postgres tables // Calls selected methods, using token holder address generated during event log conversion -func (transformer Transformer) Execute() error { - if len(transformer.Contracts) == 0 { +func (tr *transformer) Execute() error { + if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts to work with") } // Iterate through all internal contracts - for _, con := range transformer.Contracts { + for _, con := range tr.Contracts { // Update converter with current contract - transformer.Update(con) + tr.Update(con) // Iterate through contract filters and get watched event logs for eventSig, filter := range con.Filters { - watchedEvents, err := transformer.GetWatchedEvents(filter.Name) + watchedEvents, err := tr.GetWatchedEvents(filter.Name) if err != nil { return err } @@ -201,7 +181,7 @@ func (transformer Transformer) Execute() error { // Iterate over watched event logs for _, we := range watchedEvents { // Convert them to our custom log type - cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig]) + cstm, err := tr.Converter.Convert(*we, con.Events[eventSig]) if err != nil { return err } @@ -211,7 +191,7 @@ func (transformer Transformer) Execute() error { // If log is not empty, immediately persist in repo // Run this in seperate goroutine? - err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) + err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) if err != nil { return err } @@ -222,7 +202,7 @@ func (transformer Transformer) Execute() error { // poller polls select contract methods // and persists the results into custom pg tables // Run this in seperate goroutine? - if err := transformer.PollContract(*con); err != nil { + if err := tr.PollContract(*con); err != nil { return err } } @@ -230,42 +210,6 @@ func (transformer Transformer) Execute() error { return nil } -// Used to set which contract addresses and which of their events to watch -func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) { - transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to watch events for -func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) { - transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set which contract addresses and which of their methods to call -func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) { - transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to poll methods on -func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) { - transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set the block range to watch for a given address -func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) { - transformer.ContractStart[strings.ToLower(contractAddr)] = start -} - -// Used to set whether or not to persist an account address list -func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) { - transformer.CreateAddrList[strings.ToLower(contractAddr)] = on -} - -// Used to set whether or not to persist an hash list -func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) { - transformer.CreateHashList[strings.ToLower(contractAddr)] = on -} - -// Used to turn method piping on for a contract -func (transformer *Transformer) SetPiping(contractAddr string, on bool) { - transformer.Piping[strings.ToLower(contractAddr)] = on +func (tr *transformer) GetConfig() config.ContractConfig { + return tr.Config } diff --git a/pkg/omni/full/transformer/transformer_test.go b/pkg/omni/full/transformer/transformer_test.go index f25ca8f6..e302b6f5 100644 --- a/pkg/omni/full/transformer/transformer_test.go +++ b/pkg/omni/full/transformer/transformer_test.go @@ -17,160 +17,307 @@ package transformer_test import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "math/rand" + "strings" "time" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Transformer", func() { - var fakeAddress = "0x1234567890abcdef" + var db *postgres.DB + var err error + var blockChain core.BlockChain + var blockRepository repositories.BlockRepository + var ensAddr = strings.ToLower(constants.EnsContractAddress) + var tusdAddr = strings.ToLower(constants.TusdContractAddress) rand.Seed(time.Now().UnixNano()) - Describe("SetEvents", func() { - It("Sets which events to watch from the given contract address", func() { - watchedEvents := []string{"Transfer", "Mint"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, watchedEvents) - Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents)) - }) + BeforeEach(func() { + db, blockChain = test_helpers.SetupDBandBC() + blockRepository = *repositories.NewBlockRepository(db) }) - Describe("SetEventAddrs", func() { - It("Sets which account addresses to watch events for", func() { - eventAddrs := []string{"test1", "test2"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEventArgs(fakeAddress, eventAddrs) - Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs)) - }) - }) - - Describe("SetMethods", func() { - It("Sets which methods to poll at the given contract address", func() { - watchedMethods := []string{"balanceOf", "totalSupply"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethods(fakeAddress, watchedMethods) - Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods)) - }) - }) - - Describe("SetMethodAddrs", func() { - It("Sets which account addresses to poll methods against", func() { - methodAddrs := []string{"test1", "test2"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethodArgs(fakeAddress, methodAddrs) - Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs)) - }) - }) - - Describe("SetStartingBlock", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetStartingBlock(fakeAddress, 11) - Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11))) - }) - }) - - Describe("SetCreateAddrList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateAddrList(fakeAddress, true) - Expect(t.CreateAddrList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("SetCreateHashList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateHashList(fakeAddress, true) - Expect(t.CreateHashList[fakeAddress]).To(Equal(true)) - }) + AfterEach(func() { + test_helpers.TearDown(db) }) Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRetriever := &fakes.MockFullBlockRetriever{} - firstBlock := int64(1) - mostRecentBlock := int64(2) - blockRetriever.FirstBlock = firstBlock - blockRetriever.MostRecentBlock = mostRecentBlock - - parsr := &fakes.MockParser{} - fakeAbi := "fake_abi" - eventName := "Transfer" - event := types.Event{} - parsr.AbiToReturn = fakeAbi - parsr.EventName = eventName - parsr.Event = event - - pollr := &fakes.MockPoller{} - fakeContractName := "fake_contract_name" - pollr.ContractName = fakeContractName - - t := getTransformer(blockRetriever, parsr, pollr) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[fakeAddress] + c, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(true)) - Expect(c.StartingBlock).To(Equal(firstBlock)) - Expect(c.LastBlock).To(Equal(mostRecentBlock)) - Expect(c.Abi).To(Equal(fakeAbi)) - Expect(c.Name).To(Equal(fakeContractName)) - Expect(c.Address).To(Equal(fakeAddress)) + Expect(c.StartingBlock).To(Equal(int64(6194633))) + Expect(c.LastBlock).To(Equal(int64(6194634))) + Expect(c.Abi).To(Equal(constants.TusdAbiString)) + Expect(c.Name).To(Equal("TrueUSD")) + Expect(c.Address).To(Equal(tusdAddr)) }) It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { - blockRetriever := &fakes.MockFullBlockRetriever{} - blockRetriever.FirstBlockErr = fakes.FakeError - t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) }) It("Does nothing if watched events are unset", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Events = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) - err := t.Init() - - Expect(err).ToNot(HaveOccurred()) - - _, ok := t.Contracts[fakeAddress] + _, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(false)) }) }) -}) -func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { - return transformer.Transformer{ - FilterRepository: &fakes.MockFilterRepository{}, - Parser: parsr, - BlockRetriever: blockRetriever, - Poller: pollr, - Contracts: map[string]*contract.Contract{}, - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - } -} + Describe("Execute", func() { + BeforeEach(func() { + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.TransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log) + + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee")) + Expect(log.Block).To(Equal(int64(6194634))) + Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21")) + Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")) + Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146")) + }) + + It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(true)) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[""] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given methods using generated token holder address", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.BalanceOf{} + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("0")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("0")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("Fails if initialization has not been done", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + + err = t.Execute() + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Execute- against ENS registry contract", func() { + BeforeEach(func() { + blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1) + blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) + + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.NewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) + + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb")) + Expect(log.Block).To(Equal(int64(6194635))) + Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")) + Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")) + Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21")) + }) + + It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[ensAddr] + Expect(ok).To(Equal(true)) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EmittedHashes)).To(Equal(3)) + + b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + // Doesn't keep track of address since it wouldn't be used in calling the 'owner' method + _, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given methods using generated token holder address", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("It does not perist events if they do not pass the emitted arg filter", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) + Expect(err).To(HaveOccurred()) + }) + + It("If a method arg filter is applied, only those arguments are used in polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/pkg/omni/light/converter/converter.go b/pkg/omni/light/converter/converter.go index b2c0d7ac..36c18f68 100644 --- a/pkg/omni/light/converter/converter.go +++ b/pkg/omni/light/converter/converter.go @@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) { // Convert the given watched event log into a types.Log for the given event func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) returnLogs := make([]types.Log, 0, len(logs)) for _, log := range logs { values := make(map[string]interface{}) @@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in values[field.Name] = i } - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) eventsToLogs := make(map[string][]types.Log) for _, event := range events { eventsToLogs[event.Name] = make([]types.Log, 0, len(logs)) @@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E // If the log is of this event type, process it as such if event.Sig() == log.Topics[0] { values := make(map[string]interface{}) - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/omni/light/repository/header_repository.go index 87d81990..d2aeac3d 100644 --- a/pkg/omni/light/repository/header_repository.go +++ b/pkg/omni/light/repository/header_repository.go @@ -17,8 +17,8 @@ package repository import ( + "database/sql" "fmt" - "github.com/jmoiron/sqlx" "github.com/hashicorp/golang-lru" @@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) } func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { - tx, err := r.db.Beginx() + tx, err := r.db.Begin() if err != nil { return err } @@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) { return r.columns.Get(key) } -func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error { +func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) VALUES ($1, $2) ON CONFLICT (header_id) DO diff --git a/pkg/omni/light/retriever/retriever_suite_test.go b/pkg/omni/light/retriever/retriever_suite_test.go index 83a16269..5a254ef2 100644 --- a/pkg/omni/light/retriever/retriever_suite_test.go +++ b/pkg/omni/light/retriever/retriever_suite_test.go @@ -27,7 +27,7 @@ import ( func TestRetriever(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Light Block Number Retriever Suite Test") + RunSpecs(t, "Light BLock Number Retriever Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/omni/light/transformer/transformer.go b/pkg/omni/light/transformer/transformer.go index f7738c0a..b0d4f37d 100644 --- a/pkg/omni/light/transformer/transformer.go +++ b/pkg/omni/light/transformer/transformer.go @@ -18,6 +18,7 @@ package transformer import ( "errors" + "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" @@ -37,7 +38,7 @@ import ( ) // Requires a light synced vDB (headers) and a running eth node (or infura) -type Transformer struct { +type transformer struct { // Database interfaces srep.EventRepository // Holds transformed watched event log data repository.HeaderRepository // Interface for interaction with header repositories @@ -51,32 +52,12 @@ type Transformer struct { converter.Converter // Converts watched event logs into custom log poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore - // Ethereum network name; default "" is mainnet - Network string + // Store contract configuration information + Config config.ContractConfig // Store contract info as mapping to contract address Contracts map[string]*contract.Contract - // Targeted subset of events/methods - // Stored as maps of contract address to events/method names of interest - WatchedEvents map[string][]string // Default/empty event list means all are watched - WantedMethods map[string][]string // Default/empty method list means none are polled - - // Starting block number for each contract - ContractStart map[string]int64 - - // Lists of argument values to filter event or - // method data with; if empty no filter is applied - EventArgs map[string][]string - MethodArgs map[string][]string - - // Whether or not to create a list of emitted address or hashes for the contract in postgres - CreateAddrList map[string]bool - CreateHashList map[string]bool - - // Method piping on/off for a contract - Piping map[string]bool - // Internally configured transformer variables contractAddresses []string // Holds all contract addresses, for batch fetching of logs sortedEventIds map[string][]string // Map to sort event column ids by contract, for post fetch processing and persisting of logs @@ -93,26 +74,18 @@ type Transformer struct { // 4. Execute // Transformer takes in config for blockchain, database, and network id -func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer { +func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *transformer { - return &Transformer{ + return &transformer{ Poller: poller.NewPoller(bc, db, types.LightSync), Fetcher: fetcher.NewFetcher(bc), - Parser: parser.NewParser(network), + Parser: parser.NewParser(con.Network), HeaderRepository: repository.NewHeaderRepository(db), BlockRetriever: retriever.NewBlockRetriever(db), Converter: converter.NewConverter(&contract.Contract{}), Contracts: map[string]*contract.Contract{}, EventRepository: srep.NewEventRepository(db, types.LightSync), - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - Piping: map[string]bool{}, - Network: network, + Config: con, } } @@ -120,7 +93,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transf // Loops over all of the addr => filter sets // Uses parser to pull event info from abi // Use this info to generate event filters -func (tr *Transformer) Init() error { +func (tr *transformer) Init() error { // Initialize internally configured transformer settings tr.contractAddresses = make([]string, 0) // Holds all contract addresses, for batch fetching of logs tr.sortedEventIds = make(map[string][]string) // Map to sort event column ids by contract, for post fetch processing and persisting of logs @@ -130,11 +103,20 @@ func (tr *Transformer) Init() error { tr.start = 100000000000 // Hold the lowest starting block and the highest ending block // Iterate through all internal contract addresses - for contractAddr, subset := range tr.WatchedEvents { - // Get Abi - err := tr.Parser.Parse(contractAddr) - if err != nil { - return err + for contractAddr := range tr.Config.Addresses { + // Configure Abi + if tr.Config.Abis[contractAddr] == "" { + // If no abi is given in the config, this method will try fetching from internal look-up table and etherscan + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + } else { + // If we have an abi from the config, load that into the parser + err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr]) + if err != nil { + return err + } } // Get first block and most recent block number in the header repo @@ -148,40 +130,38 @@ func (tr *Transformer) Init() error { } // Set to specified range if it falls within the bounds - if firstBlock < tr.ContractStart[contractAddr] { - firstBlock = tr.ContractStart[contractAddr] + if firstBlock < tr.Config.StartingBlocks[contractAddr] { + firstBlock = tr.Config.StartingBlocks[contractAddr] } // Get contract name if it has one var name = new(string) - tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock) + tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) - // Remove any potential accidental duplicate inputs in arg filter values + // Remove any potential accidental duplicate inputs eventArgs := map[string]bool{} - for _, arg := range tr.EventArgs[contractAddr] { + for _, arg := range tr.Config.EventArgs[contractAddr] { eventArgs[arg] = true } methodArgs := map[string]bool{} - for _, arg := range tr.MethodArgs[contractAddr] { + for _, arg := range tr.Config.MethodArgs[contractAddr] { methodArgs[arg] = true } // Aggregate info into contract object and store for execution con := contract.Contract{ - Name: *name, - Network: tr.Network, - Address: contractAddr, - Abi: tr.Parser.Abi(), - ParsedAbi: tr.Parser.ParsedAbi(), - StartingBlock: firstBlock, - LastBlock: -1, - Events: tr.Parser.GetEvents(subset), - Methods: tr.Parser.GetSelectMethods(tr.WantedMethods[contractAddr]), - FilterArgs: eventArgs, - MethodArgs: methodArgs, - CreateAddrList: tr.CreateAddrList[contractAddr], - CreateHashList: tr.CreateHashList[contractAddr], - Piping: tr.Piping[contractAddr], + Name: *name, + Network: tr.Config.Network, + Address: contractAddr, + Abi: tr.Parser.Abi(), + ParsedAbi: tr.Parser.ParsedAbi(), + StartingBlock: firstBlock, + LastBlock: -1, + Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), + Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + Piping: tr.Config.Piping[contractAddr], }.Init() tr.Contracts[contractAddr] = con tr.contractAddresses = append(tr.contractAddresses, con.Address) @@ -221,7 +201,7 @@ func (tr *Transformer) Init() error { return nil } -func (tr *Transformer) Execute() error { +func (tr *transformer) Execute() error { if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts") } @@ -311,7 +291,7 @@ func (tr *Transformer) Execute() error { } // Used to poll contract methods at a given header -func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error { +func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error { for _, con := range tr.Contracts { // Skip method polling processes if no methods are specified // Also don't try to poll methods below this contract's specified starting block @@ -335,42 +315,6 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str return nil } -// Used to set which contract addresses and which of their events to watch -func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) { - tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to watch events for -func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) { - tr.EventArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set which contract addresses and which of their methods to call -func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) { - tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to poll methods on -func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) { - tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set the block range to watch for a given address -func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) { - tr.ContractStart[strings.ToLower(contractAddr)] = start -} - -// Used to set whether or not to persist an account address list -func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) { - tr.CreateAddrList[strings.ToLower(contractAddr)] = on -} - -// Used to set whether or not to persist an hash list -func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) { - tr.CreateHashList[strings.ToLower(contractAddr)] = on -} - -// Used to turn method piping on for a contract -func (tr *Transformer) SetPiping(contractAddr string, on bool) { - tr.Piping[strings.ToLower(contractAddr)] = on +func (tr *transformer) GetConfig() config.ContractConfig { + return tr.Config } diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/omni/light/transformer/transformer_test.go index 31e3c194..093cd5f2 100644 --- a/pkg/omni/light/transformer/transformer_test.go +++ b/pkg/omni/light/transformer/transformer_test.go @@ -17,150 +17,446 @@ package transformer_test import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" + "strings" + + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Transformer", func() { - var fakeAddress = "0x1234567890abcdef" + var db *postgres.DB + var err error + var blockChain core.BlockChain + var headerRepository repositories.HeaderRepository + var headerID, headerID2 int64 + var ensAddr = strings.ToLower(constants.EnsContractAddress) + var tusdAddr = strings.ToLower(constants.TusdContractAddress) - Describe("SetEvents", func() { - It("Sets which events to watch from the given contract address", func() { - watchedEvents := []string{"Transfer", "Mint"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, watchedEvents) - Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents)) - }) + BeforeEach(func() { + db, blockChain = test_helpers.SetupDBandBC() + headerRepository = repositories.NewHeaderRepository(db) }) - Describe("SetEventAddrs", func() { - It("Sets which account addresses to watch events for", func() { - eventAddrs := []string{"test1", "test2"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEventArgs(fakeAddress, eventAddrs) - Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs)) - }) - }) - - Describe("SetMethods", func() { - It("Sets which methods to poll at the given contract address", func() { - watchedMethods := []string{"balanceOf", "totalSupply"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethods(fakeAddress, watchedMethods) - Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods)) - }) - }) - - Describe("SetMethodAddrs", func() { - It("Sets which account addresses to poll methods against", func() { - methodAddrs := []string{"test1", "test2"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethodArgs(fakeAddress, methodAddrs) - Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs)) - }) - }) - - Describe("SetStartingBlock", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetStartingBlock(fakeAddress, 11) - Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11))) - }) - }) - - Describe("SetCreateAddrList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateAddrList(fakeAddress, true) - Expect(t.CreateAddrList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("SetCreateHashList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateHashList(fakeAddress, true) - Expect(t.CreateHashList[fakeAddress]).To(Equal(true)) - }) + AfterEach(func() { + test_helpers.TearDown(db) }) Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRetriever := &fakes.MockLightBlockRetriever{} - firstBlock := int64(1) - blockRetriever.FirstBlock = firstBlock - - parsr := &fakes.MockParser{} - fakeAbi := "fake_abi" - parsr.AbiToReturn = fakeAbi - - pollr := &fakes.MockPoller{} - fakeContractName := "fake_contract_name" - pollr.ContractName = fakeContractName - - t := getFakeTransformer(blockRetriever, parsr, pollr) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[fakeAddress] + c, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(true)) - Expect(c.StartingBlock).To(Equal(firstBlock)) + Expect(c.StartingBlock).To(Equal(int64(6194632))) Expect(c.LastBlock).To(Equal(int64(-1))) - Expect(c.Abi).To(Equal(fakeAbi)) - Expect(c.Name).To(Equal(fakeContractName)) - Expect(c.Address).To(Equal(fakeAddress)) + Expect(c.Abi).To(Equal(constants.TusdAbiString)) + Expect(c.Name).To(Equal("TrueUSD")) + Expect(c.Address).To(Equal(tusdAddr)) }) It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() { - blockRetriever := &fakes.MockLightBlockRetriever{} - blockRetriever.FirstBlockErr = fakes.FakeError - t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) }) - It("Does nothing if watched events are unset", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - - err := t.Init() - + It("Does nothing if nothing if no addresses are configured", func() { + headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Addresses = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() Expect(err).ToNot(HaveOccurred()) - _, ok := t.Contracts[fakeAddress] + _, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(false)) }) }) -}) + Describe("Execute- against TrueUSD contract", func() { + BeforeEach(func() { + header1, err := blockChain.GetHeaderByNumber(6791668) + Expect(err).ToNot(HaveOccurred()) + header2, err := blockChain.GetHeaderByNumber(6791669) + Expect(err).ToNot(HaveOccurred()) + header3, err := blockChain.GetHeaderByNumber(6791670) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header1) + headerID, err = headerRepository.CreateOrUpdateHeader(header2) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header3) + }) -func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { - return transformer.Transformer{ - Parser: parsr, - BlockRetriever: blockRetriever, - Poller: pollr, - HeaderRepository: &fakes.MockLightHeaderRepository{}, - Contracts: map[string]*contract.Contract{}, - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - } -} + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightTransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.HeaderID).To(Equal(headerID)) + Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) + Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) + Expect(log.Value).To(Equal("9998940000000000000000")) + }) + + It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + c, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(true)) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EmittedAddrs)).To(Equal(4)) + Expect(len(c.EmittedHashes)).To(Equal(0)) + + b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[""] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given methods using generated token holder address", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.BalanceOf{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("55849938025000000000000")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("Fails if initialization has not been done", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Execute() + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Execute- against ENS registry contract", func() { + BeforeEach(func() { + header1, err := blockChain.GetHeaderByNumber(6885695) + Expect(err).ToNot(HaveOccurred()) + header2, err := blockChain.GetHeaderByNumber(6885696) + Expect(err).ToNot(HaveOccurred()) + header3, err := blockChain.GetHeaderByNumber(6885697) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header1) + headerID, err = headerRepository.CreateOrUpdateHeader(header2) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header3) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.HeaderID).To(Equal(headerID)) + Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) + Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) + Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + }) + + It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + c, ok := t.Contracts[ensAddr] + Expect(ok).To(Equal(true)) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EmittedHashes)).To(Equal(2)) + Expect(len(c.EmittedAddrs)).To(Equal(0)) + + b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + // Doesn't keep track of address since it wouldn't be used in calling the 'owner' method + _, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given method using list of collected hashes", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("It does not persist events if they do not pass the emitted arg filter", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) + Expect(err).To(HaveOccurred()) + }) + + It("If a method arg filter is applied, only those arguments are used in polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Execute- against both ENS and TrueUSD", func() { + BeforeEach(func() { + header1, err := blockChain.GetHeaderByNumber(6791668) + Expect(err).ToNot(HaveOccurred()) + header2, err := blockChain.GetHeaderByNumber(6791669) + Expect(err).ToNot(HaveOccurred()) + header3, err := blockChain.GetHeaderByNumber(6791670) + Expect(err).ToNot(HaveOccurred()) + header4, err := blockChain.GetHeaderByNumber(6885695) + Expect(err).ToNot(HaveOccurred()) + header5, err := blockChain.GetHeaderByNumber(6885696) + Expect(err).ToNot(HaveOccurred()) + header6, err := blockChain.GetHeaderByNumber(6885697) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header1) + headerID, err = headerRepository.CreateOrUpdateHeader(header2) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header3) + headerRepository.CreateOrUpdateHeader(header4) + headerID2, err = headerRepository.CreateOrUpdateHeader(header5) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header6) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + newOwnerLog := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(newOwnerLog.HeaderID).To(Equal(headerID2)) + Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) + Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) + Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + + transferLog := test_helpers.LightTransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(transferLog.HeaderID).To(Equal(headerID)) + Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) + Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) + Expect(transferLog.Value).To(Equal("9998940000000000000000")) + }) + + It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + ens, ok := t.Contracts[ensAddr] + Expect(ok).To(Equal(true)) + tusd, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(true)) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(ens.EmittedHashes)).To(Equal(2)) + Expect(len(ens.EmittedAddrs)).To(Equal(0)) + Expect(len(tusd.EmittedAddrs)).To(Equal(4)) + Expect(len(tusd.EmittedHashes)).To(Equal(0)) + + b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + }) + + It("Polls given methods for each contract, using list of collected values", func() { + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + owner := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner) + Expect(err).ToNot(HaveOccurred()) + Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + Expect(owner.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner) + Expect(err).ToNot(HaveOccurred()) + Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(owner.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner) + Expect(err).To(HaveOccurred()) + + bal := test_helpers.BalanceOf{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal) + Expect(err).ToNot(HaveOccurred()) + Expect(bal.Balance).To(Equal("55849938025000000000000")) + Expect(bal.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/pkg/omni/shared/constants/constants.go b/pkg/omni/shared/constants/constants.go index 9ac4bd80..86012222 100644 --- a/pkg/omni/shared/constants/constants.go +++ b/pkg/omni/shared/constants/constants.go @@ -72,7 +72,6 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E" var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b" var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5" -// TODO: Consider whether these should be moved to plugins // Contract Owner var DaiContractOwner = "0x0000000000000000000000000000000000000000" var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f" diff --git a/pkg/omni/shared/contract/contract.go b/pkg/omni/shared/contract/contract.go index 19364d91..e9f06fb2 100644 --- a/pkg/omni/shared/contract/contract.go +++ b/pkg/omni/shared/contract/contract.go @@ -64,12 +64,6 @@ func (c Contract) Init() *Contract { } } - // If we are creating an address list in postgres - // we initialize the map despite what method call, if any - if c.CreateAddrList { - c.EmittedAddrs = map[interface{}]bool{} - } - return &c } diff --git a/pkg/omni/shared/helpers/test_helpers/database.go b/pkg/omni/shared/helpers/test_helpers/database.go index fb12cb3c..8a1620f7 100644 --- a/pkg/omni/shared/helpers/test_helpers/database.go +++ b/pkg/omni/shared/helpers/test_helpers/database.go @@ -106,7 +106,6 @@ type Owner struct { Address string `db:"returned"` } -// TODO: consider whether this should be moved to libraries/shared func SetupBC() core.BlockChain { infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" rawRpcClient, err := rpc.Dial(infuraIPC) @@ -114,9 +113,9 @@ func SetupBC() core.BlockChain { rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) ethClient := ethclient.NewClient(rawRpcClient) blockChainClient := client.NewEthClient(ethClient) - blockChainNode := node.MakeNode(rpcClient) + node := node.MakeNode(rpcClient) transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter) + blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) return blockChain } @@ -128,9 +127,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) { rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) ethClient := ethclient.NewClient(rawRpcClient) blockChainClient := client.NewEthClient(ethClient) - blockChainNode := node.MakeNode(rpcClient) + node := node.MakeNode(rpcClient) transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter) + blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -189,7 +188,6 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract }.Init() } -// TODO: consider whether this can be moved to plugin or libraries/shared func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) { db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -238,7 +236,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract { } func TearDown(db *postgres.DB) { - tx, err := db.Beginx() + tx, err := db.Begin() Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM blocks`) diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go b/pkg/omni/shared/helpers/test_helpers/mocks/entities.go index 9a2800c5..1829448b 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/omni/shared/helpers/test_helpers/mocks/entities.go @@ -18,6 +18,8 @@ package mocks import ( "encoding/json" + "github.com/vulcanize/vulcanizedb/pkg/config" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -249,3 +251,78 @@ var MockNewOwnerLog2 = types.Log{ }, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000af21"), } + +var ens = strings.ToLower(constants.EnsContractAddress) +var tusd = strings.ToLower(constants.TusdContractAddress) + +var TusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + tusd: true, + }, + Abis: map[string]string{ + tusd: "", + }, + Events: map[string][]string{ + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + tusd: nil, + }, + MethodArgs: map[string][]string{ + tusd: nil, + }, + EventArgs: map[string][]string{ + tusd: nil, + }, +} + +var ENSConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + }, + Abis: map[string]string{ + ens: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + }, + Methods: map[string][]string{ + ens: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + }, +} + +var ENSandTusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + tusd: true, + }, + Abis: map[string]string{ + ens: "", + tusd: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + ens: nil, + tusd: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, +} diff --git a/pkg/omni/shared/repository/event_repository.go b/pkg/omni/shared/repository/event_repository.go index a08849b4..4be2a1fe 100644 --- a/pkg/omni/shared/repository/event_repository.go +++ b/pkg/omni/shared/repository/event_repository.go @@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c // Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB) func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - tx, err := r.db.Beginx() + tx, err := r.db.Begin() if err != nil { return err } @@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB) func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - tx, err := r.db.Beginx() + tx, err := r.db.Begin() if err != nil { return err } diff --git a/pkg/omni/shared/repository/method_repository.go b/pkg/omni/shared/repository/method_repository.go index e916e045..a55adbf5 100644 --- a/pkg/omni/shared/repository/method_repository.go +++ b/pkg/omni/shared/repository/method_repository.go @@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ // Creates a custom postgres command to persist logs for the given event func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { - tx, err := r.DB.Beginx() + tx, err := r.DB.Begin() if err != nil { return err } diff --git a/pkg/omni/shared/transformer/interface.go b/pkg/omni/shared/transformer/interface.go deleted file mode 100644 index ef0d331b..00000000 --- a/pkg/omni/shared/transformer/interface.go +++ /dev/null @@ -1,32 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package transformer - -// Used to extract any/all events and a subset of method (state variable) -// data for any contract and persists it to custom postgres tables in vDB -type Transformer interface { - SetEvents(contractAddr string, filterSet []string) - SetEventArgs(contractAddr string, filterSet []string) - SetMethods(contractAddr string, filterSet []string) - SetMethodArgs(contractAddr string, filterSet []string) - SetStartingBlock(contractAddr string, start int64) - SetCreateAddrList(contractAddr string, on bool) - SetCreateHashList(contractAddr string, on bool) - SetPiping(contractAddr string, on bool) - Init() error - Execute() error -} From 5101ed4369893dcf9bc848a008fe74f59e2f3ecb Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 11:20:22 -0500 Subject: [PATCH 02/10] generic transformer and watcher interfaces for plugging in transformers built from omni pkging --- .../shared/transformer/generic_transformer.go | 31 ++++++++ libraries/shared/watcher/generic_watcher.go | 77 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 libraries/shared/transformer/generic_transformer.go create mode 100644 libraries/shared/watcher/generic_watcher.go diff --git a/libraries/shared/transformer/generic_transformer.go b/libraries/shared/transformer/generic_transformer.go new file mode 100644 index 00000000..cba448ac --- /dev/null +++ b/libraries/shared/transformer/generic_transformer.go @@ -0,0 +1,31 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package transformer + +import ( + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type GenericTransformer interface { + Init() error + Execute() error + GetConfig() config.ContractConfig +} + +type GenericTransformerInitializer func(db *postgres.DB, bc core.BlockChain) GenericTransformer diff --git a/libraries/shared/watcher/generic_watcher.go b/libraries/shared/watcher/generic_watcher.go new file mode 100644 index 00000000..2407f175 --- /dev/null +++ b/libraries/shared/watcher/generic_watcher.go @@ -0,0 +1,77 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Dynamic watcher is built with a more generic interface +// that allows offloading more of the operatinal logic to +// the transformers, allowing them to act more dynamically +// Built to work primarily with the omni pkging +package watcher + +import ( + "fmt" + + log "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" +) + +type GenericWatcher struct { + Transformers []transformer.GenericTransformer + DB *postgres.DB + BlockChain core.BlockChain +} + +func NewGenericWatcher(db *postgres.DB, bc core.BlockChain) *GenericWatcher { + return &GenericWatcher{ + DB: db, + BlockChain: bc, + } +} + +func (watcher *GenericWatcher) AddTransformers(inits interface{}) error { + initializers, ok := inits.([]transformer.GenericTransformerInitializer) + if !ok { + return fmt.Errorf("initializers of type %T, not %T", inits, []transformer.GenericTransformerInitializer{}) + } + + watcher.Transformers = make([]transformer.GenericTransformer, 0, len(initializers)) + for _, initializer := range initializers { + t := initializer(watcher.DB, watcher.BlockChain) + watcher.Transformers = append(watcher.Transformers, t) + } + + for _, transformer := range watcher.Transformers { + err := transformer.Init() + if err != nil { + log.Print("Unable to initialize transformer:", transformer.GetConfig().Name, err) + return err + } + } + return nil +} + +func (watcher *GenericWatcher) Execute(interface{}) error { + for _, transformer := range watcher.Transformers { + err := transformer.Execute() + if err != nil { + log.Error("Unable to execute transformer:", transformer.GetConfig().Name, err) + return err + } + } + return nil +} From 05c3b9bb4802f717914eafea95cc00c19e3b91ca Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 11:20:29 -0500 Subject: [PATCH 03/10] update readme; example toml --- README.md | 96 +++++++++++++++++++++++++++------------ environments/example.toml | 28 ++++++++++++ 2 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 environments/example.toml diff --git a/README.md b/README.md index 5122e0ec..aada1b37 100644 --- a/README.md +++ b/README.md @@ -154,43 +154,83 @@ Contract watchers work with a light or full sync vDB to fetch raw ethereum data A watcher is composed of at least a fetcher and a transformer or set of transformers, where a fetcher is an interface for retrieving raw Ethereum data from some source (e.g. eth_jsonrpc, IPFS) and a transformer is an interface for filtering through that raw Ethereum data to extract, process, and persist data for specific contracts or accounts. -### omniWatcher -The `omniWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. +### contractWatcher +The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. It also provides some state variable coverage by automating polling of public methods, with some restrictions. This command requires a pre-synced (full or light) vulcanizeDB (see above sections) and currently requires the contract ABI be available on etherscan or provided by the user. -To watch all events of a contract using a light synced vDB: - - Execute `./vulcanizedb omniWatcher --config --contract-address ` +This command takes a config of the form: -Or if you are using a full synced vDB, change the mode to full: - - Execute `./vulcanizedb omniWatcher --mode full --config --contract-address ` +```toml + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 -To watch contracts on a network other than mainnet, use the network flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --network ` + [client] + ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" -To watch events starting at a certain block use the starting block flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --starting-block-number <#>` + [contract] + network = "" + addresses = [ + "contractAddress1", + "contractAddress2" + ] + [contract.contractAddress1] + abi = 'ABI for contract 1' + startingBlock = 982463 + [contract.contractAddress2] + abi = 'ABI for contract 2' + events = [ + "event1", + "event2" + ] + eventArgs = [ + "arg1", + "arg2" + ] + methods = [ + "method1", + "method2" + ] + methodArgs = [ + "arg1", + "arg2" + ] + startingBlock = 4448566 + piping = true +```` -To watch only specified events use the events flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --events --events ` +- The `contract` section defines which contracts we want to watch and with which conditions. +- `network` is only necessary if the ABIs are not provided and wish to be fetched from Etherscan. + - Empty or nil string indicates mainnet + - "ropsten", "kovan", and "rinkeby" indicate their respective networks +- `addresses` lists the contract addresses we are watching and is used to load their individual configuration parameters +- `contract.` are the sub-mappings which contain the parameters specific to each contract address + - `abi` is the ABI for the contract; if none is provided the application will attempt to fetch one from Etherscan using the provided address and network + - `events` is the list of events to watch + - If this field is omitted or no events are provided then by defualt ALL events extracted from the ABI will be watched + - If event names are provided then only those events will be watched + - `eventArgs` is the list of arguments to filter events with + - If this field is omitted or no eventArgs are provided then by default watched events are not filtered by their argument values + - If eventArgs are provided then only those events which emit at least one of these values as an argument are watched + - `methods` is the list of methods to poll + - If this is omitted or no methods are provided then by default NO methods are polled + - If method names are provided then those methods will be polled, provided + 1) Method has two or less arguments + 1) Arguments are all of address or hash types + 1) Method returns a single value + - `methodArgs` is the list of arguments to limit polling methods to + - If this field is omitted or no methodArgs are provided then by default methods will be polled with every combination of the appropriately typed values that have been collected from watched events + - If methodArgs are provided then only those values will be used to poll methods + - `startingBlock` is the block we want to begin watching the contract, usually the deployment block of that contract + - `piping` is a boolean flag which indicates whether or not we want to pipe return method values forward as arguments to subsequent method calls -To watch events and poll the specified methods with any addresses and hashes emitted by the watched events utilize the methods flag: - - Execute `./vulcanizedb omniWatcher --config --contract-address --methods --methods ` +At the very minimum, for each contract address an ABI and a starting block number need to be provided (or just the starting block if the ABI can be reliably fetched from Etherscan). +With just this information we will be able to watch all events at the contract, but with no additional filters and no method polling. -To watch specified events and poll the specified method with any addresses and hashes emitted by the watched events: - - Execute `./vulcanizedb omniWatcher --config --contract-address --events --events --methods ` - -To turn on method piping so that values returned from previous method calls are cached and used as arguments in subsequent method calls: - - Execute `./vulcanizedb omniWatcher --config --piping true --contract-address --events --events --methods ` - -To watch all types of events of the contract but only persist the ones that emit one of the filtered-for argument values: - - Execute `./vulcanizedb omniWatcher --config --contract-address --event-args --event-args ` - -To watch all events of the contract but only poll the specified method with specified argument values (if they are emitted from the watched events): - - Execute `./vulcanizedb omniWatcher --config --contract-address --methods --method-args --method-args ` - -#### omniWatcher output +#### contractWatcher output Transformed events and polled method results are committed to Postgres in schemas and tables generated according to the contract abi. @@ -200,7 +240,7 @@ The 'method' and 'event' identifiers are tacked onto the end of the table names Example: -Running `./vulcanizedb omniWatcher --config --starting-block-number=5197514 --contract-address=0x8dd5fbce2f6a956c3022ba3663759011dd51e73e --events=Transfer --events=Mint --methods=balanceOf` +Running `./vulcanizedb contractWatcher --config --starting-block-number=5197514 --contract-address=0x8dd5fbce2f6a956c3022ba3663759011dd51e73e --events=Transfer --events=Mint --methods=balanceOf` watches Transfer and Mint events of the TrueUSD contract and polls its balanceOf method using the addresses we find emitted from those events It produces and populates a schema with three tables: diff --git a/environments/example.toml b/environments/example.toml new file mode 100644 index 00000000..2da5f317 --- /dev/null +++ b/environments/example.toml @@ -0,0 +1,28 @@ +[database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + +[client] + ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + +[contract] + network = "" + addresses = [ + "0x314159265dD8dbb310642f98f50C066173C1259b", + "0xdAC17F958D2ee523a2206206994597C13D831ec7" + ] + [contract.0x314159265dD8dbb310642f98f50C066173C1259b] + abi = '[{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"resolver","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"label","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setSubnodeOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"ttl","type":"uint64"}],"name":"setTTL","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"ttl","outputs":[{"name":"","type":"uint64"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"resolver","type":"address"}],"name":"setResolver","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"label","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"NewOwner","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"resolver","type":"address"}],"name":"NewResolver","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"ttl","type":"uint64"}],"name":"NewTTL","type":"event"}]' + startingBlock = 3327417 + [contract.0xdAC17F958D2ee523a2206206994597C13D831ec7] + abi = '[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_upgradedAddress","type":"address"}],"name":"deprecate","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"deprecated","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_evilUser","type":"address"}],"name":"addBlackList","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"upgradedAddress","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"balances","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"maximumFee","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"_totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_maker","type":"address"}],"name":"getBlackListStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"},{"name":"","type":"address"}],"name":"allowed","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"who","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"getOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"newBasisPoints","type":"uint256"},{"name":"newMaxFee","type":"uint256"}],"name":"setParams","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"amount","type":"uint256"}],"name":"issue","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"amount","type":"uint256"}],"name":"redeem","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"basisPointsRate","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"isBlackListed","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_clearedUser","type":"address"}],"name":"removeBlackList","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"MAX_UINT","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_blackListedUser","type":"address"}],"name":"destroyBlackFunds","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_initialSupply","type":"uint256"},{"name":"_name","type":"string"},{"name":"_symbol","type":"string"},{"name":"_decimals","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"Issue","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"Redeem","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"newAddress","type":"address"}],"name":"Deprecate","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"feeBasisPoints","type":"uint256"},{"indexed":false,"name":"maxFee","type":"uint256"}],"name":"Params","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_blackListedUser","type":"address"},{"indexed":false,"name":"_balance","type":"uint256"}],"name":"DestroyedBlackFunds","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_user","type":"address"}],"name":"AddedBlackList","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_user","type":"address"}],"name":"RemovedBlackList","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"}]' + events = [ + "Transfer", + "Issue" + ] + methods = [ + "balanceOf" + ] + startingBlock = 4634748 + piping = false \ No newline at end of file From e4e092f5425fa4d113293fb60566d48659ac4174 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 11:43:43 -0500 Subject: [PATCH 04/10] changes to plugin and commands to accomodate changes --- README.md | 32 ++++++++++++--------- cmd/compose.go | 5 ++-- cmd/composeAndExecute.go | 14 +++++++-- cmd/execute.go | 27 +++++++++++++++-- libraries/shared/watcher/generic_watcher.go | 4 +-- pkg/config/plugin.go | 5 +++- pkg/plugin/writer/writer.go | 8 +++++- 7 files changed, 70 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index aada1b37..b2546848 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,7 @@ The config provides information for composing a set of transformers: rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_event" + type = "eth_generic" repository = "github.com/account/repo" migrations = "db/migrations" rank = "0" @@ -359,6 +359,9 @@ The config provides information for composing a set of transformers: that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS) - `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go) that fetches event logs from an ETH node + - `eth_generic` indicates the transformer works with the [generic watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/generic_watcher.go) + that is made to work with [omni pkg](https://github.com/vulcanize/maker-vulcanizedb/tree/staging/pkg/omni) + based transformers which work with either a light or full sync vDB to watch events and poll public methods - `migrations` is the relative path from `repository` to the db migrations directory for the transformer - `rank` determines the order that migrations are ran, with lower ranked migrations running first - this is to help isolate any potential conflicts between transformer migrations @@ -390,14 +393,15 @@ type exporter string var Exporter exporter -func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer { - return []interface1.EventTransformerInitializer{ - transformer1.EventTransformerInitializer, - transformer2.EventTransformerInitializer, - transformer3.EventTransformerInitializer, - }, []interface1.StorageTransformerInitializer{ - transformer4.StorageTransformerInitializer, - } +func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer, []interface1.GenericTransformerInitializer { + return []interface1.TransformerInitializer{ + transformer1.TransformerInitializer, + transformer3.TransformerInitializer, + }, []interface1.StorageTransformerInitializer{ + transformer4.StorageTransformerInitializer, + }, []interface1.GenericTransformerInitializer{ + transformer2.TransformerInitializer, + } } ``` @@ -405,10 +409,12 @@ func (e exporter) Export() []interface1.EventTransformerInitializer, []interface To plug in an external transformer we need to: * Create a [package](https://github.com/vulcanize/ens_transformers/blob/working/transformers/registry/new_owner/initializer/initializer.go) -that exports a variable `EventTransformerInitializer` or `StorageTransformerInitializer` that are of type [EventTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/event_transformer.go#L33) -or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/storage_transformer.go#L31), respectively -* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go#L83) -or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/storage_watcher.go#L58) watcher execution modes +that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `GenericTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33) +or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), +or [GenericTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/transformer/generic_transformer.go#L31), respectively +* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83), +[storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53), +or [generic](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/generic_watcher.go#L68) watcher execution modes * Create db migrations to run against vulcanizeDB so that we can store the transformer output * Do not `goose fix` the transformer migrations * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields diff --git a/cmd/compose.go b/cmd/compose.go index 72f6a299..93b51af9 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -62,7 +62,7 @@ var composeCmd = &cobra.Command{ rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_event" + type = "eth_generic" repository = "github.com/account/repo" migrations = "db/migrations" rank = "0" @@ -91,7 +91,8 @@ from it and loaded into and executed over by the appropriate watcher. The type of watcher that the transformer works with is specified using the type variable for each transformer in the config. Currently there are watchers of event data from an eth node (eth_event) and storage data from an eth node -(eth_storage). +(eth_storage), and a more generic interface for accepting omni pkg based transformers +which can perform both event watching and public method polling. Transformers of different types can be ran together in the same command using a single config file or in separate command instances using different config files diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index aef951a3..b74cf280 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -62,7 +62,7 @@ var composeAndExecuteCmd = &cobra.Command{ rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_event" + type = "eth_generic" repository = "github.com/account/repo" migrations = "db/migrations" rank = "2" @@ -91,7 +91,8 @@ from it and loaded into and executed over by the appropriate watcher. The type of watcher that the transformer works with is specified using the type variable for each transformer in the config. Currently there are watchers of event data from an eth node (eth_event) and storage data from an eth node -(eth_storage). +(eth_storage), and a more generic interface for accepting omni pkg based transformers +which can perform both event watching and public method polling. Transformers of different types can be ran together in the same command using a single config file or in separate command instances using different config files @@ -150,7 +151,7 @@ func composeAndExecute() { } // Use the Exporters export method to load the EventTransformerInitializer and StorageTransformerInitializer sets - ethEventInitializers, ethStorageInitializers := exporter.Export() + ethEventInitializers, ethStorageInitializers, genericInitializers := exporter.Export() // Setup bc and db objects blockChain := getBlockChain() @@ -173,6 +174,13 @@ func composeAndExecute() { wg.Add(1) go watchEthStorage(&sw, &wg) } + + if len(genericInitializers) > 0 { + gw := watcher.NewGenericWatcher(&db, blockChain) + gw.AddTransformers(genericInitializers) + wg.Add(1) + go genericWatching(&gw, &wg) + } wg.Wait() } diff --git a/cmd/execute.go b/cmd/execute.go index da4bea4e..b76bbb59 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -100,7 +100,7 @@ func execute() { } // Use the Exporters export method to load the EventTransformerInitializer and StorageTransformerInitializer sets - ethEventInitializers, ethStorageInitializers := exporter.Export() + ethEventInitializers, ethStorageInitializers, genericInitializers := exporter.Export() // Setup bc and db objects blockChain := getBlockChain() @@ -123,6 +123,13 @@ func execute() { wg.Add(1) go watchEthStorage(&sw, &wg) } + + if len(genericInitializers) > 0 { + gw := watcher.NewGenericWatcher(&db, blockChain) + gw.AddTransformers(genericInitializers) + wg.Add(1) + go genericWatching(&gw, &wg) + } wg.Wait() } @@ -132,7 +139,7 @@ func init() { } type Exporter interface { - Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer) + Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.GenericTransformerInitializer) } func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { @@ -157,7 +164,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { defer wg.Done() - // Execute over the StorageTransformerInitializer set using the watcher + // Execute over the StorageTransformerInitializer set using the storage watcher log.Info("executing storage transformers") ticker := time.NewTicker(pollingInterval) defer ticker.Stop() @@ -168,3 +175,17 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { } } } + +func genericWatching(w *watcher.GenericWatcher, wg *syn.WaitGroup) { + defer wg.Done() + // Execute over the GenericTransformerInitializer set using the generic watcher + log.Info("executing generic transformers") + ticker := time.NewTicker(pollingInterval) + defer ticker.Stop() + for range ticker.C { + err := w.Execute(nil) + if err != nil { + // TODO Handle watcher errors in execute + } + } +} diff --git a/libraries/shared/watcher/generic_watcher.go b/libraries/shared/watcher/generic_watcher.go index 2407f175..5a49d2fd 100644 --- a/libraries/shared/watcher/generic_watcher.go +++ b/libraries/shared/watcher/generic_watcher.go @@ -36,8 +36,8 @@ type GenericWatcher struct { BlockChain core.BlockChain } -func NewGenericWatcher(db *postgres.DB, bc core.BlockChain) *GenericWatcher { - return &GenericWatcher{ +func NewGenericWatcher(db *postgres.DB, bc core.BlockChain) GenericWatcher { + return GenericWatcher{ DB: db, BlockChain: bc, } diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index dbe6a2f3..f521cb43 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -111,6 +111,7 @@ const ( UnknownTransformerType TransformerType = iota EthEvent EthStorage + EthGeneric ) func (pt TransformerType) String() string { @@ -118,9 +119,10 @@ func (pt TransformerType) String() string { "Unknown", "eth_event", "eth_storage", + "eth_generic", } - if pt > EthStorage || pt < EthEvent { + if pt > EthGeneric || pt < EthEvent { return "Unknown" } @@ -131,6 +133,7 @@ func GetTransformerType(str string) TransformerType { types := [...]TransformerType{ EthEvent, EthStorage, + EthGeneric, } for _, ty := range types { diff --git a/pkg/plugin/writer/writer.go b/pkg/plugin/writer/writer.go index 6c0b4ffd..33e8a84c 100644 --- a/pkg/plugin/writer/writer.go +++ b/pkg/plugin/writer/writer.go @@ -74,13 +74,17 @@ func (w *writer) WritePlugin() error { f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List( Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "EventTransformerInitializer"), Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"), + Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "GenericTransformerInitializer"), )).Block(Return( Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "EventTransformerInitializer").Values(code[config.EthEvent]...), Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "StorageTransformerInitializer").Values(code[config.EthStorage]...))) // Exports the collected event and storage transformer initializers + "StorageTransformerInitializer").Values(code[config.EthStorage]...), + Index().Qual( + "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", + "GenericTransformerInitializer").Values(code[config.EthGeneric]...))) // Exports the collected event and storage transformer initializers // Write code to destination file err = f.Save(goFile) @@ -100,6 +104,8 @@ func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "EventTransformerInitializer")) case config.EthStorage: code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer")) + case config.EthGeneric: + code[config.EthGeneric] = append(code[config.EthGeneric], Qual(path, "GenericTransformerInitializer")) default: return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type)) } From 073378de6dc0f79c312ac9caf4eb27cf4a5b4ec5 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 16:18:13 -0500 Subject: [PATCH 05/10] =?UTF-8?q?update=20=C2=A9=20to=202019?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/contractWatcher.go | 2 +- libraries/shared/transformer/generic_transformer.go | 2 +- libraries/shared/watcher/generic_watcher.go | 2 +- pkg/config/contract.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index 342de614..cbfe818f 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -1,5 +1,5 @@ // VulcanizeDB -// Copyright © 2018 Vulcanize +// Copyright © 2019 Vulcanize // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by diff --git a/libraries/shared/transformer/generic_transformer.go b/libraries/shared/transformer/generic_transformer.go index cba448ac..fc133b17 100644 --- a/libraries/shared/transformer/generic_transformer.go +++ b/libraries/shared/transformer/generic_transformer.go @@ -1,5 +1,5 @@ // VulcanizeDB -// Copyright © 2018 Vulcanize +// Copyright © 2019 Vulcanize // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by diff --git a/libraries/shared/watcher/generic_watcher.go b/libraries/shared/watcher/generic_watcher.go index 5a49d2fd..77b609c2 100644 --- a/libraries/shared/watcher/generic_watcher.go +++ b/libraries/shared/watcher/generic_watcher.go @@ -1,5 +1,5 @@ // VulcanizeDB -// Copyright © 2018 Vulcanize +// Copyright © 2019 Vulcanize // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by diff --git a/pkg/config/contract.go b/pkg/config/contract.go index a55933ea..a9dfefa7 100644 --- a/pkg/config/contract.go +++ b/pkg/config/contract.go @@ -1,5 +1,5 @@ // VulcanizeDB -// Copyright © 2018 Vulcanize +// Copyright © 2019 Vulcanize // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by From 24879a85aa63504982e75632a262c9b376591d8c Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 18:18:54 -0500 Subject: [PATCH 06/10] mv pkg/omni pkg/contract_watcher --- cmd/contractWatcher.go | 4 ++-- integration_test/omni_full_transformer_test.go | 8 ++++---- .../omni_light_transformer_test.go | 8 ++++---- libraries/shared/repository/repository_test.go | 2 +- .../full/converter/converter.go | 6 +++--- .../full/converter/converter_suite_test.go | 0 .../full/converter/converter_test.go | 10 +++++----- .../full/retriever/block_retriever.go | 0 .../full/retriever/block_retriever_test.go | 6 +++--- .../full/retriever/retriever_suite_test.go | 0 .../full/transformer/transformer.go | 14 +++++++------- .../full/transformer/transformer_suite_test.go | 0 .../full/transformer/transformer_test.go | 8 ++++---- .../light/converter/converter.go | 4 ++-- .../light/converter/converter_suite_test.go | 0 .../light/converter/converter_test.go | 10 +++++----- .../light/fetcher/fetcher.go | 0 .../light/fetcher/fetcher_suite_test.go | 0 .../light/fetcher/fetcher_test.go | 2 +- .../light/repository/header_repository.go | 0 .../light/repository/header_repository_test.go | 6 +++--- .../light/repository/repository_suite_test.go | 0 .../light/retriever/block_retriever.go | 0 .../light/retriever/block_retriever_test.go | 6 +++--- .../light/retriever/retriever_suite_test.go | 0 .../light/transformer/transformer.go | 18 +++++++++--------- .../transformer/transformer_suite_test.go | 0 .../light/transformer/transformer_test.go | 8 ++++---- .../shared/constants/constants.go | 2 +- .../shared/constants/interface.go | 0 .../shared/contract/contract.go | 2 +- .../shared/contract/contract_suite_test.go | 0 .../shared/contract/contract_test.go | 8 ++++---- .../shared/fetcher/fetcher.go | 0 .../shared/getter/getter_suite_test.go | 0 .../shared/getter/getter_test.go | 4 ++-- .../shared/getter/interface_getter.go | 4 ++-- .../shared/helpers/helpers.go | 0 .../shared/helpers/test_helpers/database.go | 6 +++--- .../helpers/test_helpers/mocks/entities.go | 2 +- .../helpers/test_helpers/mocks/parser.go | 2 +- .../shared/parser/parser.go | 4 ++-- .../shared/parser/parser_suite_test.go | 0 .../shared/parser/parser_test.go | 8 ++++---- .../shared/poller/poller.go | 6 +++--- .../shared/poller/poller_suite_test.go | 0 .../shared/poller/poller_test.go | 10 +++++----- .../shared/repository/event_repository.go | 4 ++-- .../shared/repository/event_repository_test.go | 18 +++++++++--------- .../shared/repository/method_repository.go | 2 +- .../repository/method_repository_test.go | 10 +++++----- .../shared/repository/repository_suite_test.go | 0 .../shared/retriever/address_retriever.go | 4 ++-- .../shared/retriever/address_retriever_test.go | 14 +++++++------- .../shared/retriever/retriever_suite_test.go | 0 .../shared/types/event.go | 0 .../shared/types/method.go | 0 .../shared/types/mode.go | 0 pkg/fakes/mock_parser.go | 2 +- pkg/fakes/mock_poller.go | 2 +- 60 files changed, 117 insertions(+), 117 deletions(-) rename pkg/{omni => contract_watcher}/full/converter/converter.go (94%) rename pkg/{omni => contract_watcher}/full/converter/converter_suite_test.go (100%) rename pkg/{omni => contract_watcher}/full/converter/converter_test.go (89%) rename pkg/{omni => contract_watcher}/full/retriever/block_retriever.go (100%) rename pkg/{omni => contract_watcher}/full/retriever/block_retriever_test.go (97%) rename pkg/{omni => contract_watcher}/full/retriever/retriever_suite_test.go (100%) rename pkg/{omni => contract_watcher}/full/transformer/transformer.go (93%) rename pkg/{omni => contract_watcher}/full/transformer/transformer_suite_test.go (100%) rename pkg/{omni => contract_watcher}/full/transformer/transformer_test.go (97%) rename pkg/{omni => contract_watcher}/light/converter/converter.go (97%) rename pkg/{omni => contract_watcher}/light/converter/converter_suite_test.go (100%) rename pkg/{omni => contract_watcher}/light/converter/converter_test.go (93%) rename pkg/{omni => contract_watcher}/light/fetcher/fetcher.go (100%) rename pkg/{omni => contract_watcher}/light/fetcher/fetcher_suite_test.go (100%) rename pkg/{omni => contract_watcher}/light/fetcher/fetcher_test.go (96%) rename pkg/{omni => contract_watcher}/light/repository/header_repository.go (100%) rename pkg/{omni => contract_watcher}/light/repository/header_repository_test.go (97%) rename pkg/{omni => contract_watcher}/light/repository/repository_suite_test.go (100%) rename pkg/{omni => contract_watcher}/light/retriever/block_retriever.go (100%) rename pkg/{omni => contract_watcher}/light/retriever/block_retriever_test.go (90%) rename pkg/{omni => contract_watcher}/light/retriever/retriever_suite_test.go (100%) rename pkg/{omni => contract_watcher}/light/transformer/transformer.go (94%) rename pkg/{omni => contract_watcher}/light/transformer/transformer_suite_test.go (100%) rename pkg/{omni => contract_watcher}/light/transformer/transformer_test.go (98%) rename pkg/{omni => contract_watcher}/shared/constants/constants.go (99%) rename pkg/{omni => contract_watcher}/shared/constants/interface.go (100%) rename pkg/{omni => contract_watcher}/shared/contract/contract.go (98%) rename pkg/{omni => contract_watcher}/shared/contract/contract_suite_test.go (100%) rename pkg/{omni => contract_watcher}/shared/contract/contract_test.go (95%) rename pkg/{omni => contract_watcher}/shared/fetcher/fetcher.go (100%) rename pkg/{omni => contract_watcher}/shared/getter/getter_suite_test.go (100%) rename pkg/{omni => contract_watcher}/shared/getter/getter_test.go (94%) rename pkg/{omni => contract_watcher}/shared/getter/interface_getter.go (96%) rename pkg/{omni => contract_watcher}/shared/helpers/helpers.go (100%) rename pkg/{omni => contract_watcher}/shared/helpers/test_helpers/database.go (97%) rename pkg/{omni => contract_watcher}/shared/helpers/test_helpers/mocks/entities.go (99%) rename pkg/{omni => contract_watcher}/shared/helpers/test_helpers/mocks/parser.go (98%) rename pkg/{omni => contract_watcher}/shared/parser/parser.go (97%) rename pkg/{omni => contract_watcher}/shared/parser/parser_suite_test.go (100%) rename pkg/{omni => contract_watcher}/shared/parser/parser_test.go (95%) rename pkg/{omni => contract_watcher}/shared/poller/poller.go (97%) rename pkg/{omni => contract_watcher}/shared/poller/poller_suite_test.go (100%) rename pkg/{omni => contract_watcher}/shared/poller/poller_test.go (97%) rename pkg/{omni => contract_watcher}/shared/repository/event_repository.go (98%) rename pkg/{omni => contract_watcher}/shared/repository/event_repository_test.go (94%) rename pkg/{omni => contract_watcher}/shared/repository/method_repository.go (99%) rename pkg/{omni => contract_watcher}/shared/repository/method_repository_test.go (95%) rename pkg/{omni => contract_watcher}/shared/repository/repository_suite_test.go (100%) rename pkg/{omni => contract_watcher}/shared/retriever/address_retriever.go (96%) rename pkg/{omni => contract_watcher}/shared/retriever/address_retriever_test.go (87%) rename pkg/{omni => contract_watcher}/shared/retriever/retriever_suite_test.go (100%) rename pkg/{omni => contract_watcher}/shared/types/event.go (100%) rename pkg/{omni => contract_watcher}/shared/types/method.go (100%) rename pkg/{omni => contract_watcher}/shared/types/mode.go (100%) diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index cbfe818f..ae1c0ca7 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -25,8 +25,8 @@ import ( "github.com/spf13/cobra" st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" - ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" + ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" + lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" "github.com/vulcanize/vulcanizedb/utils" ) diff --git a/integration_test/omni_full_transformer_test.go b/integration_test/omni_full_transformer_test.go index cdfb93ae..1da30e73 100644 --- a/integration_test/omni_full_transformer_test.go +++ b/integration_test/omni_full_transformer_test.go @@ -11,13 +11,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Omni full transformer", func() { diff --git a/integration_test/omni_light_transformer_test.go b/integration_test/omni_light_transformer_test.go index 6cc8bb93..c0efd79d 100644 --- a/integration_test/omni_light_transformer_test.go +++ b/integration_test/omni_light_transformer_test.go @@ -9,13 +9,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Omnit light transformer", func() { diff --git a/libraries/shared/repository/repository_test.go b/libraries/shared/repository/repository_test.go index 1a9977d0..ccf6257f 100644 --- a/libraries/shared/repository/repository_test.go +++ b/libraries/shared/repository/repository_test.go @@ -26,12 +26,12 @@ import ( "github.com/vulcanize/vulcanizedb/libraries/shared/constants" shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository" + r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/fakes" - r2 "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" "github.com/vulcanize/vulcanizedb/test_config" ) diff --git a/pkg/omni/full/converter/converter.go b/pkg/contract_watcher/full/converter/converter.go similarity index 94% rename from pkg/omni/full/converter/converter.go rename to pkg/contract_watcher/full/converter/converter.go index 1c615c33..47f3fa42 100644 --- a/pkg/omni/full/converter/converter.go +++ b/pkg/contract_watcher/full/converter/converter.go @@ -26,10 +26,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Converter is used to convert watched event logs to diff --git a/pkg/omni/full/converter/converter_suite_test.go b/pkg/contract_watcher/full/converter/converter_suite_test.go similarity index 100% rename from pkg/omni/full/converter/converter_suite_test.go rename to pkg/contract_watcher/full/converter/converter_suite_test.go diff --git a/pkg/omni/full/converter/converter_test.go b/pkg/contract_watcher/full/converter/converter_test.go similarity index 89% rename from pkg/omni/full/converter/converter_test.go rename to pkg/contract_watcher/full/converter/converter_test.go index 8fa7b8f5..9b7a46cb 100644 --- a/pkg/omni/full/converter/converter_test.go +++ b/pkg/contract_watcher/full/converter/converter_test.go @@ -21,11 +21,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" ) var _ = Describe("Converter", func() { diff --git a/pkg/omni/full/retriever/block_retriever.go b/pkg/contract_watcher/full/retriever/block_retriever.go similarity index 100% rename from pkg/omni/full/retriever/block_retriever.go rename to pkg/contract_watcher/full/retriever/block_retriever.go diff --git a/pkg/omni/full/retriever/block_retriever_test.go b/pkg/contract_watcher/full/retriever/block_retriever_test.go similarity index 97% rename from pkg/omni/full/retriever/block_retriever_test.go rename to pkg/contract_watcher/full/retriever/block_retriever_test.go index dcd13c35..4cc4c04e 100644 --- a/pkg/omni/full/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/full/retriever/block_retriever_test.go @@ -21,12 +21,12 @@ import ( . "github.com/onsi/gomega" "strings" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" ) var _ = Describe("Block Retriever", func() { diff --git a/pkg/omni/full/retriever/retriever_suite_test.go b/pkg/contract_watcher/full/retriever/retriever_suite_test.go similarity index 100% rename from pkg/omni/full/retriever/retriever_suite_test.go rename to pkg/contract_watcher/full/retriever/retriever_suite_test.go diff --git a/pkg/omni/full/transformer/transformer.go b/pkg/contract_watcher/full/transformer/transformer.go similarity index 93% rename from pkg/omni/full/transformer/transformer.go rename to pkg/contract_watcher/full/transformer/transformer.go index 6d51bdb1..119aa6ec 100644 --- a/pkg/omni/full/transformer/transformer.go +++ b/pkg/contract_watcher/full/transformer/transformer.go @@ -20,17 +20,17 @@ import ( "errors" "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Requires a fully synced vDB and a running eth node (or infura) diff --git a/pkg/omni/full/transformer/transformer_suite_test.go b/pkg/contract_watcher/full/transformer/transformer_suite_test.go similarity index 100% rename from pkg/omni/full/transformer/transformer_suite_test.go rename to pkg/contract_watcher/full/transformer/transformer_suite_test.go diff --git a/pkg/omni/full/transformer/transformer_test.go b/pkg/contract_watcher/full/transformer/transformer_test.go similarity index 97% rename from pkg/omni/full/transformer/transformer_test.go rename to pkg/contract_watcher/full/transformer/transformer_test.go index e302b6f5..ee6d465e 100644 --- a/pkg/omni/full/transformer/transformer_test.go +++ b/pkg/contract_watcher/full/transformer/transformer_test.go @@ -27,13 +27,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Transformer", func() { diff --git a/pkg/omni/light/converter/converter.go b/pkg/contract_watcher/light/converter/converter.go similarity index 97% rename from pkg/omni/light/converter/converter.go rename to pkg/contract_watcher/light/converter/converter.go index 36c18f68..b938c61e 100644 --- a/pkg/omni/light/converter/converter.go +++ b/pkg/contract_watcher/light/converter/converter.go @@ -28,8 +28,8 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" gethTypes "github.com/ethereum/go-ethereum/core/types" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) type Converter interface { diff --git a/pkg/omni/light/converter/converter_suite_test.go b/pkg/contract_watcher/light/converter/converter_suite_test.go similarity index 100% rename from pkg/omni/light/converter/converter_suite_test.go rename to pkg/contract_watcher/light/converter/converter_suite_test.go diff --git a/pkg/omni/light/converter/converter_test.go b/pkg/contract_watcher/light/converter/converter_test.go similarity index 93% rename from pkg/omni/light/converter/converter_test.go rename to pkg/contract_watcher/light/converter/converter_test.go index 67c9950d..fb29c556 100644 --- a/pkg/omni/light/converter/converter_test.go +++ b/pkg/contract_watcher/light/converter/converter_test.go @@ -22,11 +22,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" ) var _ = Describe("Converter", func() { diff --git a/pkg/omni/light/fetcher/fetcher.go b/pkg/contract_watcher/light/fetcher/fetcher.go similarity index 100% rename from pkg/omni/light/fetcher/fetcher.go rename to pkg/contract_watcher/light/fetcher/fetcher.go diff --git a/pkg/omni/light/fetcher/fetcher_suite_test.go b/pkg/contract_watcher/light/fetcher/fetcher_suite_test.go similarity index 100% rename from pkg/omni/light/fetcher/fetcher_suite_test.go rename to pkg/contract_watcher/light/fetcher/fetcher_suite_test.go diff --git a/pkg/omni/light/fetcher/fetcher_test.go b/pkg/contract_watcher/light/fetcher/fetcher_test.go similarity index 96% rename from pkg/omni/light/fetcher/fetcher_test.go rename to pkg/contract_watcher/light/fetcher/fetcher_test.go index 77982d84..b3b9b96c 100644 --- a/pkg/omni/light/fetcher/fetcher_test.go +++ b/pkg/contract_watcher/light/fetcher/fetcher_test.go @@ -22,9 +22,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/fetcher" ) var _ = Describe("Fetcher", func() { diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/contract_watcher/light/repository/header_repository.go similarity index 100% rename from pkg/omni/light/repository/header_repository.go rename to pkg/contract_watcher/light/repository/header_repository.go diff --git a/pkg/omni/light/repository/header_repository_test.go b/pkg/contract_watcher/light/repository/header_repository_test.go similarity index 97% rename from pkg/omni/light/repository/header_repository_test.go rename to pkg/contract_watcher/light/repository/header_repository_test.go index 6c4958b7..60aabf39 100644 --- a/pkg/omni/light/repository/header_repository_test.go +++ b/pkg/contract_watcher/light/repository/header_repository_test.go @@ -23,11 +23,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Repository", func() { diff --git a/pkg/omni/light/repository/repository_suite_test.go b/pkg/contract_watcher/light/repository/repository_suite_test.go similarity index 100% rename from pkg/omni/light/repository/repository_suite_test.go rename to pkg/contract_watcher/light/repository/repository_suite_test.go diff --git a/pkg/omni/light/retriever/block_retriever.go b/pkg/contract_watcher/light/retriever/block_retriever.go similarity index 100% rename from pkg/omni/light/retriever/block_retriever.go rename to pkg/contract_watcher/light/retriever/block_retriever.go diff --git a/pkg/omni/light/retriever/block_retriever_test.go b/pkg/contract_watcher/light/retriever/block_retriever_test.go similarity index 90% rename from pkg/omni/light/retriever/block_retriever_test.go rename to pkg/contract_watcher/light/retriever/block_retriever_test.go index d9066b92..a57852e5 100644 --- a/pkg/omni/light/retriever/block_retriever_test.go +++ b/pkg/contract_watcher/light/retriever/block_retriever_test.go @@ -20,11 +20,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Block Retriever", func() { diff --git a/pkg/omni/light/retriever/retriever_suite_test.go b/pkg/contract_watcher/light/retriever/retriever_suite_test.go similarity index 100% rename from pkg/omni/light/retriever/retriever_suite_test.go rename to pkg/contract_watcher/light/retriever/retriever_suite_test.go diff --git a/pkg/omni/light/transformer/transformer.go b/pkg/contract_watcher/light/transformer/transformer.go similarity index 94% rename from pkg/omni/light/transformer/transformer.go rename to pkg/contract_watcher/light/transformer/transformer.go index b0d4f37d..d3daa48a 100644 --- a/pkg/omni/light/transformer/transformer.go +++ b/pkg/contract_watcher/light/transformer/transformer.go @@ -24,17 +24,17 @@ import ( "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + srep "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/fetcher" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - srep "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Requires a light synced vDB (headers) and a running eth node (or infura) diff --git a/pkg/omni/light/transformer/transformer_suite_test.go b/pkg/contract_watcher/light/transformer/transformer_suite_test.go similarity index 100% rename from pkg/omni/light/transformer/transformer_suite_test.go rename to pkg/contract_watcher/light/transformer/transformer_suite_test.go diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/contract_watcher/light/transformer/transformer_test.go similarity index 98% rename from pkg/omni/light/transformer/transformer_test.go rename to pkg/contract_watcher/light/transformer/transformer_test.go index 093cd5f2..8e5902e4 100644 --- a/pkg/omni/light/transformer/transformer_test.go +++ b/pkg/contract_watcher/light/transformer/transformer_test.go @@ -25,13 +25,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Transformer", func() { diff --git a/pkg/omni/shared/constants/constants.go b/pkg/contract_watcher/shared/constants/constants.go similarity index 99% rename from pkg/omni/shared/constants/constants.go rename to pkg/contract_watcher/shared/constants/constants.go index 86012222..5545f791 100644 --- a/pkg/omni/shared/constants/constants.go +++ b/pkg/contract_watcher/shared/constants/constants.go @@ -18,9 +18,9 @@ package constants import ( "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" ) // Event enums diff --git a/pkg/omni/shared/constants/interface.go b/pkg/contract_watcher/shared/constants/interface.go similarity index 100% rename from pkg/omni/shared/constants/interface.go rename to pkg/contract_watcher/shared/constants/interface.go diff --git a/pkg/omni/shared/contract/contract.go b/pkg/contract_watcher/shared/contract/contract.go similarity index 98% rename from pkg/omni/shared/contract/contract.go rename to pkg/contract_watcher/shared/contract/contract.go index e9f06fb2..d6f62d0a 100644 --- a/pkg/omni/shared/contract/contract.go +++ b/pkg/contract_watcher/shared/contract/contract.go @@ -23,9 +23,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Contract object to hold our contract data diff --git a/pkg/omni/shared/contract/contract_suite_test.go b/pkg/contract_watcher/shared/contract/contract_suite_test.go similarity index 100% rename from pkg/omni/shared/contract/contract_suite_test.go rename to pkg/contract_watcher/shared/contract/contract_suite_test.go diff --git a/pkg/omni/shared/contract/contract_test.go b/pkg/contract_watcher/shared/contract/contract_test.go similarity index 95% rename from pkg/omni/shared/contract/contract_test.go rename to pkg/contract_watcher/shared/contract/contract_test.go index 819e6959..38a996ec 100644 --- a/pkg/omni/shared/contract/contract_test.go +++ b/pkg/contract_watcher/shared/contract/contract_test.go @@ -20,10 +20,10 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) var _ = Describe("Contract", func() { diff --git a/pkg/omni/shared/fetcher/fetcher.go b/pkg/contract_watcher/shared/fetcher/fetcher.go similarity index 100% rename from pkg/omni/shared/fetcher/fetcher.go rename to pkg/contract_watcher/shared/fetcher/fetcher.go diff --git a/pkg/omni/shared/getter/getter_suite_test.go b/pkg/contract_watcher/shared/getter/getter_suite_test.go similarity index 100% rename from pkg/omni/shared/getter/getter_suite_test.go rename to pkg/contract_watcher/shared/getter/getter_suite_test.go diff --git a/pkg/omni/shared/getter/getter_test.go b/pkg/contract_watcher/shared/getter/getter_test.go similarity index 94% rename from pkg/omni/shared/getter/getter_test.go rename to pkg/contract_watcher/shared/getter/getter_test.go index 730767db..c84560ce 100644 --- a/pkg/omni/shared/getter/getter_test.go +++ b/pkg/contract_watcher/shared/getter/getter_test.go @@ -22,12 +22,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/getter" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/getter" ) var _ = Describe("Interface Getter", func() { diff --git a/pkg/omni/shared/getter/interface_getter.go b/pkg/contract_watcher/shared/getter/interface_getter.go similarity index 96% rename from pkg/omni/shared/getter/interface_getter.go rename to pkg/contract_watcher/shared/getter/interface_getter.go index 70fc8273..8aee7d03 100644 --- a/pkg/omni/shared/getter/interface_getter.go +++ b/pkg/contract_watcher/shared/getter/interface_getter.go @@ -17,9 +17,9 @@ package getter import ( + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher" "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/fetcher" ) type InterfaceGetter interface { diff --git a/pkg/omni/shared/helpers/helpers.go b/pkg/contract_watcher/shared/helpers/helpers.go similarity index 100% rename from pkg/omni/shared/helpers/helpers.go rename to pkg/contract_watcher/shared/helpers/helpers.go diff --git a/pkg/omni/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go similarity index 97% rename from pkg/omni/shared/helpers/test_helpers/database.go rename to pkg/contract_watcher/shared/helpers/test_helpers/database.go index 8a1620f7..69ed26a3 100644 --- a/pkg/omni/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -24,6 +24,9 @@ import ( . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" @@ -31,9 +34,6 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) type TransferLog struct { diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go similarity index 99% rename from pkg/omni/shared/helpers/test_helpers/mocks/entities.go rename to pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go index 1829448b..debcc8dc 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go @@ -25,9 +25,9 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" ) var TransferBlock1 = core.Block{ diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/parser.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go similarity index 98% rename from pkg/omni/shared/helpers/test_helpers/mocks/parser.go rename to pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go index 5dd5d2e8..6ea0948d 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/parser.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go @@ -19,8 +19,8 @@ package mocks import ( "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Mock parser diff --git a/pkg/omni/shared/parser/parser.go b/pkg/contract_watcher/shared/parser/parser.go similarity index 97% rename from pkg/omni/shared/parser/parser.go rename to pkg/contract_watcher/shared/parser/parser.go index 22686bfb..cc3da5bd 100644 --- a/pkg/omni/shared/parser/parser.go +++ b/pkg/contract_watcher/shared/parser/parser.go @@ -22,9 +22,9 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) // Parser is used to fetch and parse contract ABIs diff --git a/pkg/omni/shared/parser/parser_suite_test.go b/pkg/contract_watcher/shared/parser/parser_suite_test.go similarity index 100% rename from pkg/omni/shared/parser/parser_suite_test.go rename to pkg/contract_watcher/shared/parser/parser_suite_test.go diff --git a/pkg/omni/shared/parser/parser_test.go b/pkg/contract_watcher/shared/parser/parser_test.go similarity index 95% rename from pkg/omni/shared/parser/parser_test.go rename to pkg/contract_watcher/shared/parser/parser_test.go index a3ee5bfc..95b065cb 100644 --- a/pkg/omni/shared/parser/parser_test.go +++ b/pkg/contract_watcher/shared/parser/parser_test.go @@ -21,11 +21,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/geth" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Parser", func() { diff --git a/pkg/omni/shared/poller/poller.go b/pkg/contract_watcher/shared/poller/poller.go similarity index 97% rename from pkg/omni/shared/poller/poller.go rename to pkg/contract_watcher/shared/poller/poller.go index 7964e8bf..0a38316c 100644 --- a/pkg/omni/shared/poller/poller.go +++ b/pkg/contract_watcher/shared/poller/poller.go @@ -26,11 +26,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) type Poller interface { diff --git a/pkg/omni/shared/poller/poller_suite_test.go b/pkg/contract_watcher/shared/poller/poller_suite_test.go similarity index 100% rename from pkg/omni/shared/poller/poller_suite_test.go rename to pkg/contract_watcher/shared/poller/poller_suite_test.go diff --git a/pkg/omni/shared/poller/poller_test.go b/pkg/contract_watcher/shared/poller/poller_test.go similarity index 97% rename from pkg/omni/shared/poller/poller_test.go rename to pkg/contract_watcher/shared/poller/poller_test.go index 7b2df18d..c4e06762 100644 --- a/pkg/omni/shared/poller/poller_test.go +++ b/pkg/contract_watcher/shared/poller/poller_test.go @@ -23,13 +23,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Poller", func() { diff --git a/pkg/omni/shared/repository/event_repository.go b/pkg/contract_watcher/shared/repository/event_repository.go similarity index 98% rename from pkg/omni/shared/repository/event_repository.go rename to pkg/contract_watcher/shared/repository/event_repository.go index 4be2a1fe..742ca9ad 100644 --- a/pkg/omni/shared/repository/event_repository.go +++ b/pkg/contract_watcher/shared/repository/event_repository.go @@ -23,9 +23,9 @@ import ( "github.com/hashicorp/golang-lru" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) const ( diff --git a/pkg/omni/shared/repository/event_repository_test.go b/pkg/contract_watcher/shared/repository/event_repository_test.go similarity index 94% rename from pkg/omni/shared/repository/event_repository_test.go rename to pkg/contract_watcher/shared/repository/event_repository_test.go index aee6fe40..d6c3bff9 100644 --- a/pkg/omni/shared/repository/event_repository_test.go +++ b/pkg/contract_watcher/shared/repository/event_repository_test.go @@ -26,17 +26,17 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + fc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" + lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" + sr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" - fc "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - lc "github.com/vulcanize/vulcanizedb/pkg/omni/light/converter" - lr "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" - sr "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Repository", func() { diff --git a/pkg/omni/shared/repository/method_repository.go b/pkg/contract_watcher/shared/repository/method_repository.go similarity index 99% rename from pkg/omni/shared/repository/method_repository.go rename to pkg/contract_watcher/shared/repository/method_repository.go index a55adbf5..4747f141 100644 --- a/pkg/omni/shared/repository/method_repository.go +++ b/pkg/contract_watcher/shared/repository/method_repository.go @@ -23,8 +23,8 @@ import ( "github.com/hashicorp/golang-lru" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) const methodCacheSize = 1000 diff --git a/pkg/omni/shared/repository/method_repository_test.go b/pkg/contract_watcher/shared/repository/method_repository_test.go similarity index 95% rename from pkg/omni/shared/repository/method_repository_test.go rename to pkg/contract_watcher/shared/repository/method_repository_test.go index 54a665c2..b50f1ee5 100644 --- a/pkg/omni/shared/repository/method_repository_test.go +++ b/pkg/contract_watcher/shared/repository/method_repository_test.go @@ -23,12 +23,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Repository", func() { diff --git a/pkg/omni/shared/repository/repository_suite_test.go b/pkg/contract_watcher/shared/repository/repository_suite_test.go similarity index 100% rename from pkg/omni/shared/repository/repository_suite_test.go rename to pkg/contract_watcher/shared/repository/repository_suite_test.go diff --git a/pkg/omni/shared/retriever/address_retriever.go b/pkg/contract_watcher/shared/retriever/address_retriever.go similarity index 96% rename from pkg/omni/shared/retriever/address_retriever.go rename to pkg/contract_watcher/shared/retriever/address_retriever.go index 61e2b939..6acdf72b 100644 --- a/pkg/omni/shared/retriever/address_retriever.go +++ b/pkg/contract_watcher/shared/retriever/address_retriever.go @@ -18,14 +18,14 @@ package retriever import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "strings" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" ) // Address retriever is used to retrieve the addresses associated with a contract diff --git a/pkg/omni/shared/retriever/address_retriever_test.go b/pkg/contract_watcher/shared/retriever/address_retriever_test.go similarity index 87% rename from pkg/omni/shared/retriever/address_retriever_test.go rename to pkg/contract_watcher/shared/retriever/address_retriever_test.go index fd19a8f1..7ff4ced7 100644 --- a/pkg/omni/shared/retriever/address_retriever_test.go +++ b/pkg/contract_watcher/shared/retriever/address_retriever_test.go @@ -21,15 +21,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/repository" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/retriever" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/converter" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/retriever" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var mockEvent = core.WatchedEvent{ diff --git a/pkg/omni/shared/retriever/retriever_suite_test.go b/pkg/contract_watcher/shared/retriever/retriever_suite_test.go similarity index 100% rename from pkg/omni/shared/retriever/retriever_suite_test.go rename to pkg/contract_watcher/shared/retriever/retriever_suite_test.go diff --git a/pkg/omni/shared/types/event.go b/pkg/contract_watcher/shared/types/event.go similarity index 100% rename from pkg/omni/shared/types/event.go rename to pkg/contract_watcher/shared/types/event.go diff --git a/pkg/omni/shared/types/method.go b/pkg/contract_watcher/shared/types/method.go similarity index 100% rename from pkg/omni/shared/types/method.go rename to pkg/contract_watcher/shared/types/method.go diff --git a/pkg/omni/shared/types/mode.go b/pkg/contract_watcher/shared/types/mode.go similarity index 100% rename from pkg/omni/shared/types/mode.go rename to pkg/contract_watcher/shared/types/mode.go diff --git a/pkg/fakes/mock_parser.go b/pkg/fakes/mock_parser.go index cac8dd9c..e705f413 100644 --- a/pkg/fakes/mock_parser.go +++ b/pkg/fakes/mock_parser.go @@ -2,7 +2,7 @@ package fakes import ( "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) type MockParser struct { diff --git a/pkg/fakes/mock_poller.go b/pkg/fakes/mock_poller.go index f1a1caec..ab7287ee 100644 --- a/pkg/fakes/mock_poller.go +++ b/pkg/fakes/mock_poller.go @@ -1,7 +1,7 @@ package fakes import ( - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" ) type MockPoller struct { From 37e581c7ec8e975443e403c55f0ae2324ea818cc Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 19:02:04 -0500 Subject: [PATCH 07/10] PR#72 from public repo- https://github.com/vulcanize/vulcanizedb/pull/72- also needed to finish pluggin in ENS record transformer --- ...contract_watcher_full_transformer_test.go} | 2 +- ...ontract_watcher_light_transformer_test.go} | 59 ++++--------- .../light/repository/header_repository.go | 46 ++++++++-- .../repository/header_repository_test.go | 88 +++++++++++++++---- .../light/transformer/transformer.go | 2 + .../light/transformer/transformer_test.go | 57 ++++-------- .../helpers/test_helpers/mocks/entities.go | 7 ++ pkg/contract_watcher/shared/poller/poller.go | 15 ++-- pkg/geth/contract.go | 6 +- 9 files changed, 165 insertions(+), 117 deletions(-) rename integration_test/{omni_full_transformer_test.go => contract_watcher_full_transformer_test.go} (99%) rename integration_test/{omni_light_transformer_test.go => contract_watcher_light_transformer_test.go} (88%) diff --git a/integration_test/omni_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go similarity index 99% rename from integration_test/omni_full_transformer_test.go rename to integration_test/contract_watcher_full_transformer_test.go index 1da30e73..cecf11eb 100644 --- a/integration_test/omni_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -20,7 +20,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" ) -var _ = Describe("Omni full transformer", func() { +var _ = Describe("contractWatcher full transformer", func() { var db *postgres.DB var err error var blockChain core.BlockChain diff --git a/integration_test/omni_light_transformer_test.go b/integration_test/contract_watcher_light_transformer_test.go similarity index 88% rename from integration_test/omni_light_transformer_test.go rename to integration_test/contract_watcher_light_transformer_test.go index c0efd79d..3ca326c3 100644 --- a/integration_test/omni_light_transformer_test.go +++ b/integration_test/contract_watcher_light_transformer_test.go @@ -18,12 +18,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" ) -var _ = Describe("Omnit light transformer", func() { +var _ = Describe("contractWatcher light transformer", func() { var db *postgres.DB var err error var blockChain core.BlockChain var headerRepository repositories.HeaderRepository - var headerID, headerID2 int64 + var headerID int64 var ensAddr = strings.ToLower(constants.EnsContractAddress) var tusdAddr = strings.ToLower(constants.TusdContractAddress) @@ -292,26 +292,12 @@ var _ = Describe("Omnit light transformer", func() { Describe("Execute- against both ENS and TrueUSD", func() { BeforeEach(func() { - header1, err := blockChain.GetHeaderByNumber(6791668) - Expect(err).ToNot(HaveOccurred()) - header2, err := blockChain.GetHeaderByNumber(6791669) - Expect(err).ToNot(HaveOccurred()) - header3, err := blockChain.GetHeaderByNumber(6791670) - Expect(err).ToNot(HaveOccurred()) - header4, err := blockChain.GetHeaderByNumber(6885695) - Expect(err).ToNot(HaveOccurred()) - header5, err := blockChain.GetHeaderByNumber(6885696) - Expect(err).ToNot(HaveOccurred()) - header6, err := blockChain.GetHeaderByNumber(6885697) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header1) - headerID, err = headerRepository.CreateOrUpdateHeader(header2) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header3) - headerRepository.CreateOrUpdateHeader(header4) - headerID2, err = headerRepository.CreateOrUpdateHeader(header5) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header6) + for i := 6885692; i < 6885702; i++ { + header, err := blockChain.GetHeaderByNumber(int64(i)) + Expect(err).ToNot(HaveOccurred()) + _, err = headerRepository.CreateOrUpdateHeader(header) + Expect(err).ToNot(HaveOccurred()) + } }) It("Transforms watched contract data into custom repositories", func() { @@ -325,7 +311,6 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) Expect(err).ToNot(HaveOccurred()) // We don't know vulcID, so compare individual fields instead of complete structures - Expect(newOwnerLog.HeaderID).To(Equal(headerID2)) Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) @@ -334,10 +319,9 @@ var _ = Describe("Omnit light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog) Expect(err).ToNot(HaveOccurred()) // We don't know vulcID, so compare individual fields instead of complete structures - Expect(transferLog.HeaderID).To(Equal(headerID)) - Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) - Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) - Expect(transferLog.Value).To(Equal("9998940000000000000000")) + Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")) + Expect(transferLog.To).To(Equal("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")) + Expect(transferLog.Value).To(Equal("2800000000000000000000")) }) It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { @@ -358,7 +342,7 @@ var _ = Describe("Omnit light transformer", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(ens.EmittedHashes)).To(Equal(2)) Expect(len(ens.EmittedAddrs)).To(Equal(0)) - Expect(len(tusd.EmittedAddrs)).To(Equal(4)) + Expect(len(tusd.EmittedAddrs)).To(Equal(2)) Expect(len(tusd.EmittedHashes)).To(Equal(0)) b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] @@ -369,21 +353,16 @@ var _ = Describe("Omnit light transformer", func() { Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + b, ok = tusd.EmittedAddrs[common.HexToAddress("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) + _, ok = tusd.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] + Expect(ok).To(Equal(false)) }) It("Polls given methods for each contract, using list of collected values", func() { @@ -414,12 +393,12 @@ var _ = Describe("Omnit light transformer", func() { Expect(err).To(HaveOccurred()) bal := test_helpers.BalanceOf{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).ToNot(HaveOccurred()) - Expect(bal.Balance).To(Equal("55849938025000000000000")) + Expect(bal.Balance).To(Equal("1954436000000000000000")) Expect(bal.TokenName).To(Equal("TrueUSD")) - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/contract_watcher/light/repository/header_repository.go b/pkg/contract_watcher/light/repository/header_repository.go index d2aeac3d..47437f13 100644 --- a/pkg/contract_watcher/light/repository/header_repository.go +++ b/pkg/contract_watcher/light/repository/header_repository.go @@ -53,6 +53,7 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository { } } +// Adds a checked_header column for the provided column id func (r *headerRepository) AddCheckColumn(id string) error { // Check cache to see if column already exists before querying pg _, ok := r.columns.Get(id) @@ -73,6 +74,7 @@ func (r *headerRepository) AddCheckColumn(id string) error { return nil } +// Adds a checked_header column for all of the provided column ids func (r *headerRepository) AddCheckColumns(ids []string) error { var err error baseQuery := "ALTER TABLE public.checked_headers" @@ -96,6 +98,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error { return err } +// Marks the header checked for the provided column id func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { _, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`) VALUES ($1, $2) @@ -105,6 +108,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error { return err } +// Marks the header checked for all of the provided column ids func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error { pgStr := "INSERT INTO public.checked_headers (header_id, " for _, id := range ids { @@ -124,6 +128,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) return err } +// Marks all of the provided headers checked for each of the provided column ids func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { tx, err := r.db.Begin() if err != nil { @@ -154,6 +159,7 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [ return tx.Commit() } +// Returns missing headers for the provided checked_headers column id func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) { var result []core.Header var query string @@ -165,7 +171,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber WHERE (header_id ISNULL OR checked_headers.` + id + `=0) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers @@ -174,13 +180,14 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } - return result, err + return contiguousHeaders(result, startingBlockNumber), err } +// Returns missing headers for all of the provided checked_headers column ids func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) { var result []core.Header var query string @@ -196,21 +203,42 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock if endingBlockNumber == -1 { endStr := `) AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { endStr := `) AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } - return result, err + return contiguousHeaders(result, startingBlockNumber), err } +// Takes in an ordered sequence of headers and returns only the first contiguous segment +// Enforce continuity with previous segment with the appropriate startingBlockNumber +func contiguousHeaders(headers []core.Header, startingBlockNumber int64) []core.Header { + if len(headers) < 1 { + return headers + } + previousHeader := headers[0].BlockNumber + if previousHeader != startingBlockNumber { + return []core.Header{} + } + for i := 1; i < len(headers); i++ { + previousHeader++ + if headers[i].BlockNumber != previousHeader { + return headers[:i] + } + } + + return headers +} + +// Returns headers that have been checked for all of the provided event ids but not for the provided method ids func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) { var result []core.Header var query string @@ -231,14 +259,14 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock if endingBlockNumber == -1 { endStr := `AND headers.block_number >= $1 AND headers.eth_node_fingerprint = $2 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { endStr := `AND headers.block_number >= $1 AND headers.block_number <= $2 AND headers.eth_node_fingerprint = $3 - ORDER BY headers.block_number` + ORDER BY headers.block_number LIMIT 100` query = baseQuery + endStr err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } @@ -246,10 +274,12 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock return result, err } +// Check the repositories column id cache for a value func (r *headerRepository) CheckCache(key string) (interface{}, bool) { return r.columns.Get(key) } +// Used to mark a header checked as part of some external transaction so as to group into one commit func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) VALUES ($1, $2) diff --git a/pkg/contract_watcher/light/repository/header_repository_test.go b/pkg/contract_watcher/light/repository/header_repository_test.go index 60aabf39..1ca2e1ac 100644 --- a/pkg/contract_watcher/light/repository/header_repository_test.go +++ b/pkg/contract_watcher/light/repository/header_repository_test.go @@ -32,6 +32,7 @@ import ( var _ = Describe("Repository", func() { var db *postgres.DB + var bc core.BlockChain var omniHeaderRepo repository.HeaderRepository // omni/light header repository var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository var eventIDs = []string{ @@ -46,7 +47,7 @@ var _ = Describe("Repository", func() { } BeforeEach(func() { - db, _ = test_helpers.SetupDBandBC() + db, bc = test_helpers.SetupDBandBC() omniHeaderRepo = repository.NewHeaderRepository(db) coreHeaderRepo = repositories.NewHeaderRepository(db) }) @@ -120,7 +121,7 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) }) @@ -130,7 +131,7 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -142,12 +143,24 @@ var _ = Describe("Repository", func() { Expect(h3.BlockNumber).To(Equal(int64(6194634))) }) + It("Returns only contiguous chunks of headers", func() { + addDiscontinuousHeaders(coreHeaderRepo) + err := omniHeaderRepo.AddCheckColumns(eventIDs) + Expect(err).ToNot(HaveOccurred()) + + missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(2)) + Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) + Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633))) + }) + It("Fails if eventID does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - _, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, "notEventId") + _, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, "notEventId") Expect(err).To(HaveOccurred()) }) }) @@ -158,14 +171,14 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -174,18 +187,42 @@ var _ = Describe("Repository", func() { err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194633, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) + It("Returns only contiguous chunks of headers", func() { + addDiscontinuousHeaders(coreHeaderRepo) + err := omniHeaderRepo.AddCheckColumns(eventIDs) + Expect(err).ToNot(HaveOccurred()) + + missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(2)) + Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) + Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633))) + }) + + It("Returns at most 100 headers", func() { + add102Headers(coreHeaderRepo, bc) + err := omniHeaderRepo.AddCheckColumns(eventIDs) + Expect(err).ToNot(HaveOccurred()) + + missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194733, eventIDs) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingHeaders)).To(Equal(100)) + Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) + Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633))) + }) + It("Fails if one of the eventIDs does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) err := omniHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) badEventIDs := append(eventIDs, "notEventId") - _, err = omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, badEventIDs) + _, err = omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, badEventIDs) Expect(err).To(HaveOccurred()) }) }) @@ -196,7 +233,7 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -204,7 +241,7 @@ var _ = Describe("Repository", func() { err = omniHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err = omniHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) @@ -214,7 +251,7 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -222,7 +259,7 @@ var _ = Describe("Repository", func() { err = omniHeaderRepo.MarkHeaderChecked(headerID, "notEventId") Expect(err).To(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) }) @@ -234,7 +271,7 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194630, 6194635, eventIDs) + missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -242,7 +279,7 @@ var _ = Describe("Repository", func() { err = omniHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err = omniHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) @@ -261,7 +298,7 @@ var _ = Describe("Repository", func() { for _, id := range methodIDs { err := omniHeaderRepo.AddCheckColumn(id) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id) + missingHeaders, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, id) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) } @@ -269,7 +306,7 @@ var _ = Describe("Repository", func() { err := omniHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range methodIDs { - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194630, 6194635, id) + missingHeaders, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, id) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(0)) } @@ -286,7 +323,7 @@ var _ = Describe("Repository", func() { Expect(err).ToNot(HaveOccurred()) } - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194630, 6194635, eventIDs[0]) + missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -301,7 +338,7 @@ var _ = Describe("Repository", func() { Expect(err).ToNot(HaveOccurred()) } - intersectionHeaders, err := omniHeaderRepo.MissingMethodsCheckedEventsIntersection(6194630, 6194635, methodIDs, eventIDs) + intersectionHeaders, err := omniHeaderRepo.MissingMethodsCheckedEventsIntersection(6194632, 6194635, methodIDs, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(intersectionHeaders)).To(Equal(1)) Expect(intersectionHeaders[0].Id).To(Equal(headerID2)) @@ -315,3 +352,18 @@ func addHeaders(coreHeaderRepo repositories.HeaderRepository) { coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2) coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3) } + +func addDiscontinuousHeaders(coreHeaderRepo repositories.HeaderRepository) { + coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1) + coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2) + coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4) +} + +func add102Headers(coreHeaderRepo repositories.HeaderRepository, blockChain core.BlockChain) { + for i := 6194632; i < 6194733; i++ { + header, err := blockChain.GetHeaderByNumber(int64(i)) + Expect(err).ToNot(HaveOccurred()) + _, err = coreHeaderRepo.CreateOrUpdateHeader(header) + Expect(err).ToNot(HaveOccurred()) + } +} diff --git a/pkg/contract_watcher/light/transformer/transformer.go b/pkg/contract_watcher/light/transformer/transformer.go index d3daa48a..080ca040 100644 --- a/pkg/contract_watcher/light/transformer/transformer.go +++ b/pkg/contract_watcher/light/transformer/transformer.go @@ -285,6 +285,8 @@ func (tr *transformer) Execute() error { if err != nil { return err } + + tr.start = header.BlockNumber + 1 } return nil diff --git a/pkg/contract_watcher/light/transformer/transformer_test.go b/pkg/contract_watcher/light/transformer/transformer_test.go index 8e5902e4..9fe3d3a1 100644 --- a/pkg/contract_watcher/light/transformer/transformer_test.go +++ b/pkg/contract_watcher/light/transformer/transformer_test.go @@ -39,7 +39,7 @@ var _ = Describe("Transformer", func() { var err error var blockChain core.BlockChain var headerRepository repositories.HeaderRepository - var headerID, headerID2 int64 + var headerID int64 var ensAddr = strings.ToLower(constants.EnsContractAddress) var tusdAddr = strings.ToLower(constants.TusdContractAddress) @@ -328,26 +328,12 @@ var _ = Describe("Transformer", func() { Describe("Execute- against both ENS and TrueUSD", func() { BeforeEach(func() { - header1, err := blockChain.GetHeaderByNumber(6791668) - Expect(err).ToNot(HaveOccurred()) - header2, err := blockChain.GetHeaderByNumber(6791669) - Expect(err).ToNot(HaveOccurred()) - header3, err := blockChain.GetHeaderByNumber(6791670) - Expect(err).ToNot(HaveOccurred()) - header4, err := blockChain.GetHeaderByNumber(6885695) - Expect(err).ToNot(HaveOccurred()) - header5, err := blockChain.GetHeaderByNumber(6885696) - Expect(err).ToNot(HaveOccurred()) - header6, err := blockChain.GetHeaderByNumber(6885697) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header1) - headerID, err = headerRepository.CreateOrUpdateHeader(header2) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header3) - headerRepository.CreateOrUpdateHeader(header4) - headerID2, err = headerRepository.CreateOrUpdateHeader(header5) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header6) + for i := 6885692; i < 6885702; i++ { + header, err := blockChain.GetHeaderByNumber(int64(i)) + Expect(err).ToNot(HaveOccurred()) + _, err = headerRepository.CreateOrUpdateHeader(header) + Expect(err).ToNot(HaveOccurred()) + } }) It("Transforms watched contract data into custom repositories", func() { @@ -361,7 +347,6 @@ var _ = Describe("Transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) Expect(err).ToNot(HaveOccurred()) // We don't know vulcID, so compare individual fields instead of complete structures - Expect(newOwnerLog.HeaderID).To(Equal(headerID2)) Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) @@ -370,10 +355,9 @@ var _ = Describe("Transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog) Expect(err).ToNot(HaveOccurred()) // We don't know vulcID, so compare individual fields instead of complete structures - Expect(transferLog.HeaderID).To(Equal(headerID)) - Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) - Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) - Expect(transferLog.Value).To(Equal("9998940000000000000000")) + Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")) + Expect(transferLog.To).To(Equal("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")) + Expect(transferLog.Value).To(Equal("2800000000000000000000")) }) It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { @@ -394,7 +378,7 @@ var _ = Describe("Transformer", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(ens.EmittedHashes)).To(Equal(2)) Expect(len(ens.EmittedAddrs)).To(Equal(0)) - Expect(len(tusd.EmittedAddrs)).To(Equal(4)) + Expect(len(tusd.EmittedAddrs)).To(Equal(2)) Expect(len(tusd.EmittedHashes)).To(Equal(0)) b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] @@ -405,21 +389,16 @@ var _ = Describe("Transformer", func() { Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + b, ok = tusd.EmittedAddrs[common.HexToAddress("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) + _, ok = tusd.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] + Expect(ok).To(Equal(false)) }) It("Polls given methods for each contract, using list of collected values", func() { @@ -450,12 +429,12 @@ var _ = Describe("Transformer", func() { Expect(err).To(HaveOccurred()) bal := test_helpers.BalanceOf{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).ToNot(HaveOccurred()) - Expect(bal.Balance).To(Equal("55849938025000000000000")) + Expect(bal.Balance).To(Equal("1954436000000000000000")) Expect(bal.TokenName).To(Equal("TrueUSD")) - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go index debcc8dc..e65db708 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go @@ -183,6 +183,13 @@ var MockHeader3 = core.Header{ Timestamp: "50000030", } +var MockHeader4 = core.Header{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + BlockNumber: 6194635, + Raw: rawFakeHeader, + Timestamp: "50000030", +} + var MockTransferLog1 = types.Log{ Index: 1, Address: common.HexToAddress(constants.TusdContractAddress), diff --git a/pkg/contract_watcher/shared/poller/poller.go b/pkg/contract_watcher/shared/poller/poller.go index 0a38316c..ae4f1974 100644 --- a/pkg/contract_watcher/shared/poller/poller.go +++ b/pkg/contract_watcher/shared/poller/poller.go @@ -54,7 +54,9 @@ func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *po func (p *poller) PollContract(con contract.Contract) error { for i := con.StartingBlock; i <= con.LastBlock; i++ { - p.PollContractAt(con, i) + if err := p.PollContractAt(con, i); err != nil { + return err + } } return nil @@ -98,7 +100,6 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error { if err != nil { return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } - strOut, err := stringify(out) if err != nil { return err @@ -138,8 +139,8 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { if len(args) == 0 { // If we haven't collected any args by now we can't call the method return nil } - results := make([]types.Result, 0, len(args)) + results := make([]types.Result, 0, len(args)) for arg := range args { in := []interface{}{arg} strIn := []interface{}{contract.StringifyArg(arg)} @@ -160,7 +161,6 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { result.Output = strOut results = append(results, result) } - // Persist result set as batch err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) if err != nil { @@ -204,7 +204,6 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { } results := make([]types.Result, 0, len(firstArgs)*len(secondArgs)) - for arg1 := range firstArgs { for arg2 := range secondArgs { in := []interface{}{arg1, arg2} @@ -215,18 +214,15 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { if err != nil { return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } - strOut, err := stringify(out) if err != nil { return err } - p.cache(out) result.Output = strOut result.Inputs = strIn results = append(results, result) - } } @@ -243,9 +239,8 @@ func (p *poller) FetchContractData(contractAbi, contractAddress, method string, return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber) } -// This is used to cache an method return value if method piping is turned on +// This is used to cache a method return value if method piping is turned on func (p *poller) cache(out interface{}) { - // Cache returned value if piping is turned on if p.contract.Piping { switch out.(type) { case common.Hash: diff --git a/pkg/geth/contract.go b/pkg/geth/contract.go index f59cab5f..b5f968ec 100644 --- a/pkg/geth/contract.go +++ b/pkg/geth/contract.go @@ -43,7 +43,11 @@ func (blockChain *BlockChain) FetchContractData(abiJSON string, address string, if err != nil { return err } - output, err := blockChain.callContract(address, input, big.NewInt(blockNumber)) + var bn *big.Int + if blockNumber > 0 { + bn = big.NewInt(blockNumber) + } + output, err := blockChain.callContract(address, input, bn) if err != nil { return err } From 37fc03860549c3f122c3acffabbb2aaa4e7dc034 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 13 Mar 2019 11:14:35 -0500 Subject: [PATCH 08/10] fixes for review comments --- README.md | 54 ++- cmd/compose.go | 7 +- cmd/composeAndExecute.go | 19 +- cmd/contractWatcher.go | 2 +- cmd/execute.go | 20 +- environments/example.toml | 10 +- .../contract_watcher_full_transformer_test.go | 47 +- ...contract_watcher_light_transformer_test.go | 56 ++- ...transformer.go => contract_transformer.go} | 4 +- ...generic_watcher.go => contract_watcher.go} | 24 +- pkg/config/contract.go | 163 ++++--- pkg/config/plugin.go | 28 +- .../full/converter/converter.go | 14 +- .../full/converter/converter_test.go | 12 +- .../full/transformer/transformer.go | 22 +- .../full/transformer/transformer_test.go | 333 +++---------- .../light/converter/converter.go | 16 +- .../light/converter/converter_test.go | 15 +- .../repository/header_repository_test.go | 110 ++--- .../light/retriever/retriever_suite_test.go | 2 +- .../light/transformer/transformer.go | 48 +- .../light/transformer/transformer_test.go | 448 ++---------------- .../shared/contract/contract.go | 2 +- .../helpers/test_helpers/mocks/entities.go | 58 +-- .../shared/helpers/test_helpers/test_data.go | 109 +++++ .../repository/event_repository_test.go | 6 +- .../retriever/address_retriever_test.go | 3 +- pkg/contract_watcher/shared/types/event.go | 2 +- pkg/contract_watcher/shared/types/mode.go | 4 +- pkg/fakes/mock_parser.go | 5 +- pkg/plugin/writer/writer.go | 8 +- 31 files changed, 634 insertions(+), 1017 deletions(-) rename libraries/shared/transformer/{generic_transformer.go => contract_transformer.go} (87%) rename libraries/shared/watcher/{generic_watcher.go => contract_watcher.go} (71%) create mode 100644 pkg/contract_watcher/shared/helpers/test_helpers/test_data.go diff --git a/README.md b/README.md index b2546848..e9787b6c 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,17 @@ and a transformer is an interface for filtering through that raw Ethereum data t ### contractWatcher The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. -It also provides some state variable coverage by automating polling of public methods, with some restrictions. +It also provides some state variable coverage by automating polling of public methods, with some restrictions: +1. The method must have 2 or less arguments +2. The method's arguments must all be of type address or bytes32 (hash) +3. The method must return a single value -This command requires a pre-synced (full or light) vulcanizeDB (see above sections) and currently requires the contract ABI be available on etherscan or provided by the user. +This command operates in two modes- `light` and `full`- which require a light or full-synced vulcanizeDB, respectively. + +This command requires the contract ABI be available on Etherscan if it is not provided in the config file by the user. + +If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `light` or `full` mode. +Otherwise, when operating in `light` mode, we only need to connect to a full node to fetch event logs. This command takes a config of the form: @@ -240,10 +248,26 @@ The 'method' and 'event' identifiers are tacked onto the end of the table names Example: -Running `./vulcanizedb contractWatcher --config --starting-block-number=5197514 --contract-address=0x8dd5fbce2f6a956c3022ba3663759011dd51e73e --events=Transfer --events=Mint --methods=balanceOf` -watches Transfer and Mint events of the TrueUSD contract and polls its balanceOf method using the addresses we find emitted from those events +Running `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light` -It produces and populates a schema with three tables: +Runs our contract watcher in light mode, configured to watch the contracts specified in the config file. Note that +by default we operate in `light` mode but the flag is included here to demonstrate its use. + +The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E). + +Because the ENS Registry is configured with only an ABI and a starting block, we will watch all events for this contract and poll none of its methods. Note that the ENS Registry is an example +of a contract which does not have its ABI available over Etherscan and must have it included in the config file. + +The TrueUSD contract is configured with two events (`Transfer` and `Mint`) and a single method (`balanceOf`), as such it will watch these two events and use any addresses it collects emitted from them +to poll the `balanceOf` method with those addresses at every block. Note that we do not provide an ABI for TrueUSD as its ABI can be fetched from Etherscan. + +For the ENS contract, it produces and populates a schema with four tables" +`light_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event` +`light_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event` +`light_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event` +`light_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event` + +For the TrusUSD contract, it produces and populates a schema with three tables: `light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event` `light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event` @@ -276,7 +300,9 @@ Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method" | who_ | character varying(66) | | not null | | extended | | | | returned | numeric | | not null | | main | | | -The addition of '_' after table names is to prevent collisions with reserved Postgres words +The addition of '_' after table names is to prevent collisions with reserved Postgres words. + +Also notice that the contract address used for the schema name has been down-cased. ### composeAndExecute The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers. @@ -359,9 +385,9 @@ The config provides information for composing a set of transformers: that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS) - `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go) that fetches event logs from an ETH node - - `eth_generic` indicates the transformer works with the [generic watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/generic_watcher.go) - that is made to work with [omni pkg](https://github.com/vulcanize/maker-vulcanizedb/tree/staging/pkg/omni) - based transformers which work with either a light or full sync vDB to watch events and poll public methods + - `eth_contract` indicates the transformer works with the [contract watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/generic_watcher.go) + that is made to work with [contract_watcher pkg](https://github.com/vulcanize/maker-vulcanizedb/tree/staging/pkg/omni) + based transformers which work with either a light or full sync vDB to watch events and poll public methods ([example](https://github.com/vulcanize/ens_transformers/blob/working/transformers/domain_records/transformer.go)) - `migrations` is the relative path from `repository` to the db migrations directory for the transformer - `rank` determines the order that migrations are ran, with lower ranked migrations running first - this is to help isolate any potential conflicts between transformer migrations @@ -393,13 +419,13 @@ type exporter string var Exporter exporter -func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer, []interface1.GenericTransformerInitializer { +func (e exporter) Export() []interface1.EventTransformerInitializer, []interface1.StorageTransformerInitializer, []interface1.ContractTransformerInitializer { return []interface1.TransformerInitializer{ transformer1.TransformerInitializer, transformer3.TransformerInitializer, }, []interface1.StorageTransformerInitializer{ transformer4.StorageTransformerInitializer, - }, []interface1.GenericTransformerInitializer{ + }, []interface1.ContractTransformerInitializer{ transformer2.TransformerInitializer, } } @@ -409,12 +435,12 @@ func (e exporter) Export() []interface1.EventTransformerInitializer, []interface To plug in an external transformer we need to: * Create a [package](https://github.com/vulcanize/ens_transformers/blob/working/transformers/registry/new_owner/initializer/initializer.go) -that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `GenericTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33) +that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33) or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), -or [GenericTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/transformer/generic_transformer.go#L31), respectively +or [ContractTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/transformer/contract_transformer.go#L31), respectively * Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83), [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53), -or [generic](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/generic_watcher.go#L68) watcher execution modes +or [contract](https://github.com/vulcanize/maker-vulcanizedb/blob/omni_update/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes * Create db migrations to run against vulcanizeDB so that we can store the transformer output * Do not `goose fix` the transformer migrations * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields diff --git a/cmd/compose.go b/cmd/compose.go index 93b51af9..28de4af1 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -62,7 +62,7 @@ var composeCmd = &cobra.Command{ rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_generic" + type = "eth_contract" repository = "github.com/account/repo" migrations = "db/migrations" rank = "0" @@ -91,8 +91,9 @@ from it and loaded into and executed over by the appropriate watcher. The type of watcher that the transformer works with is specified using the type variable for each transformer in the config. Currently there are watchers of event data from an eth node (eth_event) and storage data from an eth node -(eth_storage), and a more generic interface for accepting omni pkg based transformers -which can perform both event watching and public method polling. +(eth_storage), and a more generic interface for accepting contract_watcher pkg +based transformers which can perform both event watching and public method +polling (eth_contract). Transformers of different types can be ran together in the same command using a single config file or in separate command instances using different config files diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index b74cf280..c6aa318b 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -62,7 +62,7 @@ var composeAndExecuteCmd = &cobra.Command{ rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_generic" + type = "eth_contract" repository = "github.com/account/repo" migrations = "db/migrations" rank = "2" @@ -91,8 +91,9 @@ from it and loaded into and executed over by the appropriate watcher. The type of watcher that the transformer works with is specified using the type variable for each transformer in the config. Currently there are watchers of event data from an eth node (eth_event) and storage data from an eth node -(eth_storage), and a more generic interface for accepting omni pkg based transformers -which can perform both event watching and public method polling. +(eth_storage), and a more generic interface for accepting contract_watcher pkg +based transformers which can perform both event watching and public method +polling (eth_contract). Transformers of different types can be ran together in the same command using a single config file or in separate command instances using different config files @@ -150,8 +151,8 @@ func composeAndExecute() { os.Exit(1) } - // Use the Exporters export method to load the EventTransformerInitializer and StorageTransformerInitializer sets - ethEventInitializers, ethStorageInitializers, genericInitializers := exporter.Export() + // Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets + ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export() // Setup bc and db objects blockChain := getBlockChain() @@ -175,11 +176,11 @@ func composeAndExecute() { go watchEthStorage(&sw, &wg) } - if len(genericInitializers) > 0 { - gw := watcher.NewGenericWatcher(&db, blockChain) - gw.AddTransformers(genericInitializers) + if len(ethContractInitializers) > 0 { + gw := watcher.NewContractWatcher(&db, blockChain) + gw.AddTransformers(ethContractInitializers) wg.Add(1) - go genericWatching(&gw, &wg) + go contractWatching(&gw, &wg) } wg.Wait() } diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index ae1c0ca7..9305249f 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -94,7 +94,7 @@ func contractWatcher() { blockChain := getBlockChain() db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - var t st.GenericTransformer + var t st.ContractTransformer con := config.ContractConfig{} con.PrepConfig() switch mode { diff --git a/cmd/execute.go b/cmd/execute.go index b76bbb59..3010db59 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -99,8 +99,8 @@ func execute() { os.Exit(1) } - // Use the Exporters export method to load the EventTransformerInitializer and StorageTransformerInitializer sets - ethEventInitializers, ethStorageInitializers, genericInitializers := exporter.Export() + // Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets + ethEventInitializers, ethStorageInitializers, ethContractInitializers := exporter.Export() // Setup bc and db objects blockChain := getBlockChain() @@ -124,11 +124,11 @@ func execute() { go watchEthStorage(&sw, &wg) } - if len(genericInitializers) > 0 { - gw := watcher.NewGenericWatcher(&db, blockChain) - gw.AddTransformers(genericInitializers) + if len(ethContractInitializers) > 0 { + gw := watcher.NewContractWatcher(&db, blockChain) + gw.AddTransformers(ethContractInitializers) wg.Add(1) - go genericWatching(&gw, &wg) + go contractWatching(&gw, &wg) } wg.Wait() } @@ -139,7 +139,7 @@ func init() { } type Exporter interface { - Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.GenericTransformerInitializer) + Export() ([]transformer.EventTransformerInitializer, []transformer.StorageTransformerInitializer, []transformer.ContractTransformerInitializer) } func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { @@ -176,10 +176,10 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { } } -func genericWatching(w *watcher.GenericWatcher, wg *syn.WaitGroup) { +func contractWatching(w *watcher.ContractWatcher, wg *syn.WaitGroup) { defer wg.Done() - // Execute over the GenericTransformerInitializer set using the generic watcher - log.Info("executing generic transformers") + // Execute over the ContractTransformerInitializer set using the contract watcher + log.Info("executing contract_watcher transformers") ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { diff --git a/environments/example.toml b/environments/example.toml index 2da5f317..66522f05 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -1,5 +1,5 @@ [database] - name = "vulcanize_public" + name = "vulcanize_infura" hostname = "localhost" port = 5432 @@ -10,13 +10,12 @@ network = "" addresses = [ "0x314159265dD8dbb310642f98f50C066173C1259b", - "0xdAC17F958D2ee523a2206206994597C13D831ec7" + "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E" ] [contract.0x314159265dD8dbb310642f98f50C066173C1259b] abi = '[{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"resolver","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"label","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setSubnodeOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"ttl","type":"uint64"}],"name":"setTTL","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"ttl","outputs":[{"name":"","type":"uint64"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"resolver","type":"address"}],"name":"setResolver","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"label","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"NewOwner","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"resolver","type":"address"}],"name":"NewResolver","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"ttl","type":"uint64"}],"name":"NewTTL","type":"event"}]' startingBlock = 3327417 - [contract.0xdAC17F958D2ee523a2206206994597C13D831ec7] - abi = '[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_upgradedAddress","type":"address"}],"name":"deprecate","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"deprecated","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_evilUser","type":"address"}],"name":"addBlackList","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"upgradedAddress","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"balances","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"maximumFee","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"_totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_maker","type":"address"}],"name":"getBlackListStatus","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"},{"name":"","type":"address"}],"name":"allowed","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"who","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"getOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"newBasisPoints","type":"uint256"},{"name":"newMaxFee","type":"uint256"}],"name":"setParams","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"amount","type":"uint256"}],"name":"issue","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"amount","type":"uint256"}],"name":"redeem","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"basisPointsRate","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"isBlackListed","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_clearedUser","type":"address"}],"name":"removeBlackList","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"MAX_UINT","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_blackListedUser","type":"address"}],"name":"destroyBlackFunds","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_initialSupply","type":"uint256"},{"name":"_name","type":"string"},{"name":"_symbol","type":"string"},{"name":"_decimals","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"Issue","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"Redeem","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"newAddress","type":"address"}],"name":"Deprecate","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"feeBasisPoints","type":"uint256"},{"indexed":false,"name":"maxFee","type":"uint256"}],"name":"Params","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_blackListedUser","type":"address"},{"indexed":false,"name":"_balance","type":"uint256"}],"name":"DestroyedBlackFunds","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_user","type":"address"}],"name":"AddedBlackList","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"_user","type":"address"}],"name":"RemovedBlackList","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"}]' + [contract.0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E] events = [ "Transfer", "Issue" @@ -24,5 +23,4 @@ methods = [ "balanceOf" ] - startingBlock = 4634748 - piping = false \ No newline at end of file + startingBlock = 5197514 \ No newline at end of file diff --git a/integration_test/contract_watcher_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go index cecf11eb..03e174b9 100644 --- a/integration_test/contract_watcher_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -42,7 +42,7 @@ var _ = Describe("contractWatcher full transformer", func() { It("Initializes transformer's contract objects", func() { blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -55,6 +55,28 @@ var _ = Describe("contractWatcher full transformer", func() { Expect(c.Name).To(Equal("TrueUSD")) Expect(c.Address).To(Equal(tusdAddr)) }) + + It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) + }) + + It("Does nothing if watched events are unset", func() { + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Events = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no filters created")) + + _, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(false)) + }) }) Describe("Execute", func() { @@ -64,7 +86,7 @@ var _ = Describe("contractWatcher full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -84,7 +106,7 @@ var _ = Describe("contractWatcher full transformer", func() { It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { var testConf config.ContractConfig - testConf = mocks.TusdConfig + testConf = test_helpers.TusdConfig testConf.Methods = map[string][]string{ tusdAddr: {"balanceOf"}, } @@ -121,7 +143,7 @@ var _ = Describe("contractWatcher full transformer", func() { It("Polls given methods using generated token holder address", func() { var testConf config.ContractConfig - testConf = mocks.TusdConfig + testConf = test_helpers.TusdConfig testConf.Methods = map[string][]string{ tusdAddr: {"balanceOf"}, } @@ -146,13 +168,15 @@ var _ = Describe("contractWatcher full transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("transformer has no initialized contracts to work with")) }) }) @@ -163,7 +187,7 @@ var _ = Describe("contractWatcher full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -184,7 +208,7 @@ var _ = Describe("contractWatcher full transformer", func() { It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.Methods = map[string][]string{ ensAddr: {"owner"}, } @@ -214,7 +238,7 @@ var _ = Describe("contractWatcher full transformer", func() { It("Polls given methods using generated token holder address", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.Methods = map[string][]string{ ensAddr: {"owner"}, } @@ -238,11 +262,12 @@ var _ = Describe("contractWatcher full transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("It does not perist events if they do not pass the emitted arg filter", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.EventArgs = map[string][]string{ ensAddr: {"fake_filter_value"}, } @@ -257,11 +282,12 @@ var _ = Describe("contractWatcher full transformer", func() { log := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not exist")) }) It("If a method arg filter is applied, only those arguments are used in polling", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.MethodArgs = map[string][]string{ ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, } @@ -283,6 +309,7 @@ var _ = Describe("contractWatcher full transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) }) }) diff --git a/integration_test/contract_watcher_light_transformer_test.go b/integration_test/contract_watcher_light_transformer_test.go index 3ca326c3..45323c10 100644 --- a/integration_test/contract_watcher_light_transformer_test.go +++ b/integration_test/contract_watcher_light_transformer_test.go @@ -2,13 +2,13 @@ package integration import ( "fmt" - "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" @@ -40,7 +40,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Initializes transformer's contract objects", func() { headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -53,6 +53,27 @@ var _ = Describe("contractWatcher light transformer", func() { Expect(c.Name).To(Equal("TrueUSD")) Expect(c.Address).To(Equal(tusdAddr)) }) + + It("Fails to initialize if first and block cannot be fetched from vDB headers table", func() { + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) + }) + + It("Does nothing if nothing if no addresses are configured", func() { + headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + var testConf config.ContractConfig + testConf = test_helpers.TusdConfig + testConf.Addresses = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + _, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(false)) + }) }) Describe("Execute- against TrueUSD contract", func() { @@ -70,7 +91,7 @@ var _ = Describe("contractWatcher light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -88,7 +109,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { var testConf config.ContractConfig - testConf = mocks.TusdConfig + testConf = test_helpers.TusdConfig testConf.Methods = map[string][]string{ tusdAddr: {"balanceOf"}, } @@ -133,7 +154,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Polls given methods using generated token holder address", func() { var testConf config.ContractConfig - testConf = mocks.TusdConfig + testConf = test_helpers.TusdConfig testConf.Methods = map[string][]string{ tusdAddr: {"balanceOf"}, } @@ -151,12 +172,14 @@ var _ = Describe("contractWatcher light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("transformer has no initialized contracts")) }) }) @@ -175,7 +198,7 @@ var _ = Describe("contractWatcher light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -193,7 +216,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.Methods = map[string][]string{ ensAddr: {"owner"}, } @@ -222,7 +245,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Polls given method using list of collected hashes", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.Methods = map[string][]string{ ensAddr: {"owner"}, } @@ -245,11 +268,12 @@ var _ = Describe("contractWatcher light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) It("It does not persist events if they do not pass the emitted arg filter", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.EventArgs = map[string][]string{ ensAddr: {"fake_filter_value"}, } @@ -262,11 +286,12 @@ var _ = Describe("contractWatcher light transformer", func() { log := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not exist")) }) It("If a method arg filter is applied, only those arguments are used in polling", func() { var testConf config.ContractConfig - testConf = mocks.ENSConfig + testConf = test_helpers.ENSConfig testConf.MethodArgs = map[string][]string{ ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, } @@ -287,6 +312,7 @@ var _ = Describe("contractWatcher light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) }) @@ -301,7 +327,7 @@ var _ = Describe("contractWatcher light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db) + t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -326,7 +352,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { var testConf config.ContractConfig - testConf = mocks.ENSandTusdConfig + testConf = test_helpers.ENSandTusdConfig testConf.Methods = map[string][]string{ ensAddr: {"owner"}, tusdAddr: {"balanceOf"}, @@ -367,7 +393,7 @@ var _ = Describe("contractWatcher light transformer", func() { It("Polls given methods for each contract, using list of collected values", func() { var testConf config.ContractConfig - testConf = mocks.ENSandTusdConfig + testConf = test_helpers.ENSandTusdConfig testConf.Methods = map[string][]string{ ensAddr: {"owner"}, tusdAddr: {"balanceOf"}, @@ -391,6 +417,7 @@ var _ = Describe("contractWatcher light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) bal := test_helpers.BalanceOf{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal) @@ -400,6 +427,7 @@ var _ = Describe("contractWatcher light transformer", func() { err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no rows in result set")) }) }) }) diff --git a/libraries/shared/transformer/generic_transformer.go b/libraries/shared/transformer/contract_transformer.go similarity index 87% rename from libraries/shared/transformer/generic_transformer.go rename to libraries/shared/transformer/contract_transformer.go index fc133b17..14b8ab97 100644 --- a/libraries/shared/transformer/generic_transformer.go +++ b/libraries/shared/transformer/contract_transformer.go @@ -22,10 +22,10 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type GenericTransformer interface { +type ContractTransformer interface { Init() error Execute() error GetConfig() config.ContractConfig } -type GenericTransformerInitializer func(db *postgres.DB, bc core.BlockChain) GenericTransformer +type ContractTransformerInitializer func(db *postgres.DB, bc core.BlockChain) ContractTransformer diff --git a/libraries/shared/watcher/generic_watcher.go b/libraries/shared/watcher/contract_watcher.go similarity index 71% rename from libraries/shared/watcher/generic_watcher.go rename to libraries/shared/watcher/contract_watcher.go index 77b609c2..880dfb5b 100644 --- a/libraries/shared/watcher/generic_watcher.go +++ b/libraries/shared/watcher/contract_watcher.go @@ -14,10 +14,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -// Dynamic watcher is built with a more generic interface -// that allows offloading more of the operatinal logic to +// Contract watcher is built with a more generic interface +// that allows offloading more of the operational logic to // the transformers, allowing them to act more dynamically -// Built to work primarily with the omni pkging +// Built to work primarily with the contract_watcher packaging package watcher import ( @@ -30,26 +30,26 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) -type GenericWatcher struct { - Transformers []transformer.GenericTransformer +type ContractWatcher struct { + Transformers []transformer.ContractTransformer DB *postgres.DB BlockChain core.BlockChain } -func NewGenericWatcher(db *postgres.DB, bc core.BlockChain) GenericWatcher { - return GenericWatcher{ +func NewContractWatcher(db *postgres.DB, bc core.BlockChain) ContractWatcher { + return ContractWatcher{ DB: db, BlockChain: bc, } } -func (watcher *GenericWatcher) AddTransformers(inits interface{}) error { - initializers, ok := inits.([]transformer.GenericTransformerInitializer) +func (watcher *ContractWatcher) AddTransformers(inits interface{}) error { + initializers, ok := inits.([]transformer.ContractTransformerInitializer) if !ok { - return fmt.Errorf("initializers of type %T, not %T", inits, []transformer.GenericTransformerInitializer{}) + return fmt.Errorf("initializers of type %T, not %T", inits, []transformer.ContractTransformerInitializer{}) } - watcher.Transformers = make([]transformer.GenericTransformer, 0, len(initializers)) + watcher.Transformers = make([]transformer.ContractTransformer, 0, len(initializers)) for _, initializer := range initializers { t := initializer(watcher.DB, watcher.BlockChain) watcher.Transformers = append(watcher.Transformers, t) @@ -65,7 +65,7 @@ func (watcher *GenericWatcher) AddTransformers(inits interface{}) error { return nil } -func (watcher *GenericWatcher) Execute(interface{}) error { +func (watcher *ContractWatcher) Execute(interface{}) error { for _, transformer := range watcher.Transformers { err := transformer.Execute() if err != nil { diff --git a/pkg/config/contract.go b/pkg/config/contract.go index a9dfefa7..a2a09d2f 100644 --- a/pkg/config/contract.go +++ b/pkg/config/contract.go @@ -71,7 +71,7 @@ func (oc *ContractConfig) PrepConfig() { oc.Addresses = make(map[string]bool, len(addrs)) oc.Abis = make(map[string]string, len(addrs)) oc.Methods = make(map[string][]string, len(addrs)) - oc.EventArgs = make(map[string][]string, len(addrs)) + oc.Events = make(map[string][]string, len(addrs)) oc.MethodArgs = make(map[string][]string, len(addrs)) oc.EventArgs = make(map[string][]string, len(addrs)) oc.StartingBlocks = make(map[string]int64, len(addrs)) @@ -86,92 +86,131 @@ func (oc *ContractConfig) PrepConfig() { transformer := viper.GetStringMap("contract." + addr) // Get and check abi - abi, abiOK := transformer["abi"] - if !abiOK || abi == nil { - log.Fatal(addr, "transformer config is missing `abi` value") - } - abiRef, abiOK := abi.(string) + var abi string + _, abiOK := transformer["abi"] if !abiOK { - log.Fatal(addr, "transformer `events` not of type []string") + log.Warnf("contract %s not configured with an ABI, will attempt to fetch it from Etherscan\r\n", addr) + } else { + abiInterface := transformer["abi"] + abi, abiOK = abiInterface.(string) + if !abiOK { + log.Fatal(addr, "transformer `abi` not of type []string") + } } - oc.Abis[strings.ToLower(addr)] = abiRef + oc.Abis[strings.ToLower(addr)] = abi // Get and check events - events, eventsOK := transformer["events"] - if !eventsOK || events == nil { - log.Fatal(addr, "transformer config is missing `events` value") - } - eventsRef, eventsOK := events.([]string) + events := make([]string, 0) + _, eventsOK := transformer["events"] if !eventsOK { - log.Fatal(addr, "transformer `events` not of type []string") + log.Warnf("contract %s not configured with a list of events to watch, will watch all events\r\n", addr) + events = []string{} + } else { + eventsInterface := transformer["events"] + eventsI, eventsOK := eventsInterface.([]interface{}) + if !eventsOK { + log.Fatal(addr, "transformer `events` not of type []string\r\n") + } + for _, strI := range eventsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `events` not of type []string\r\n") + } + events = append(events, str) + } } - if eventsRef == nil { - eventsRef = []string{} - } - oc.Events[strings.ToLower(addr)] = eventsRef + oc.Events[strings.ToLower(addr)] = events // Get and check methods - methods, methodsOK := transformer["methods"] - if !methodsOK || methods == nil { - log.Fatal(addr, "transformer config is missing `methods` value") - } - methodsRef, methodsOK := methods.([]string) + methods := make([]string, 0) + _, methodsOK := transformer["methods"] if !methodsOK { - log.Fatal(addr, "transformer `methods` not of type []string") + log.Warnf("contract %s not configured with a list of methods to poll, will not poll any methods\r\n", addr) + methods = []string{} + } else { + methodsInterface := transformer["methods"] + methodsI, methodsOK := methodsInterface.([]interface{}) + if !methodsOK { + log.Fatal(addr, "transformer `methods` not of type []string\r\n") + } + for _, strI := range methodsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `methods` not of type []string\r\n") + } + methods = append(methods, str) + } } - if methodsRef == nil { - methodsRef = []string{} - } - oc.Methods[strings.ToLower(addr)] = methodsRef + oc.Methods[strings.ToLower(addr)] = methods // Get and check eventArgs - eventArgs, eventArgsOK := transformer["eventArgs"] - if !eventArgsOK || eventArgs == nil { - log.Fatal(addr, "transformer config is missing `eventArgs` value") - } - eventArgsRef, eventArgsOK := eventArgs.([]string) + eventArgs := make([]string, 0) + _, eventArgsOK := transformer["eventArgs"] if !eventArgsOK { - log.Fatal(addr, "transformer `eventArgs` not of type []string") + log.Warnf("contract %s not configured with a list of event arguments to filter for, will not filter events for specific emitted values\r\n", addr) + eventArgs = []string{} + } else { + eventArgsInterface := transformer["eventArgs"] + eventArgsI, eventArgsOK := eventArgsInterface.([]interface{}) + if !eventArgsOK { + log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n") + } + for _, strI := range eventArgsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n") + } + eventArgs = append(eventArgs, str) + } } - if eventArgsRef == nil { - eventArgsRef = []string{} - } - oc.EventArgs[strings.ToLower(addr)] = eventArgsRef + oc.EventArgs[strings.ToLower(addr)] = eventArgs // Get and check methodArgs - methodArgs, methodArgsOK := transformer["methodArgs"] - if !methodArgsOK || methodArgs == nil { - log.Fatal(addr, "transformer config is missing `methodArgs` value") - } - methodArgsRef, methodArgsOK := methodArgs.([]string) + methodArgs := make([]string, 0) + _, methodArgsOK := transformer["methodArgs"] if !methodArgsOK { - log.Fatal(addr, "transformer `methodArgs` not of type []string") + log.Warnf("contract %s not configured with a list of method argument values to poll with, will poll methods with all available arguments\r\n", addr) + methodArgs = []string{} + } else { + methodArgsInterface := transformer["methodArgs"] + methodArgsI, methodArgsOK := methodArgsInterface.([]interface{}) + if !methodArgsOK { + log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n") + } + for _, strI := range methodArgsI { + str, strOK := strI.(string) + if !strOK { + log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n") + } + methodArgs = append(methodArgs, str) + } } - if methodArgsRef == nil { - methodArgsRef = []string{} - } - oc.MethodArgs[strings.ToLower(addr)] = methodArgsRef + oc.MethodArgs[strings.ToLower(addr)] = methodArgs // Get and check startingBlock - start, startOK := transformer["startingBlock"] - if !startOK || start == nil { - log.Fatal(addr, "transformer config is missing `startingBlock` value") - } - startRef, startOK := start.(int64) + startInterface, startOK := transformer["startingblock"] if !startOK { - log.Fatal(addr, "transformer `startingBlock` not of type int") + log.Fatal(addr, "transformer config is missing `startingBlock` value\r\n") } - oc.StartingBlocks[strings.ToLower(addr)] = startRef + start, startOK := startInterface.(int64) + if !startOK { + log.Fatal(addr, "transformer `startingBlock` not of type int\r\n") + } + oc.StartingBlocks[strings.ToLower(addr)] = start // Get pipping - pipe, pipeOK := transformer["pipping"] - if !pipeOK || pipe == nil { - log.Fatal(addr, "transformer config is missing `pipping` value") - } - pipeRef, pipeOK := pipe.(bool) + var piping bool + _, pipeOK := transformer["piping"] if !pipeOK { - log.Fatal(addr, "transformer `piping` not of type bool") + log.Warnf("contract %s does not have its `piping` set, by default piping is turned off\r\n", addr) + piping = false + } else { + pipingInterface := transformer["piping"] + piping, pipeOK = pipingInterface.(bool) + if !pipeOK { + log.Fatal(addr, "transformer `piping` not of type bool\r\n") + } } - oc.Piping[strings.ToLower(addr)] = pipeRef + oc.Piping[strings.ToLower(addr)] = piping } } diff --git a/pkg/config/plugin.go b/pkg/config/plugin.go index f521cb43..fad07cbc 100644 --- a/pkg/config/plugin.go +++ b/pkg/config/plugin.go @@ -41,13 +41,13 @@ type Transformer struct { RepositoryPath string } -func (c *Plugin) GetPluginPaths() (string, string, error) { - path, err := helpers.CleanPath(c.FilePath) +func (pluginConfig *Plugin) GetPluginPaths() (string, string, error) { + path, err := helpers.CleanPath(pluginConfig.FilePath) if err != nil { return "", "", err } - name := strings.Split(c.FileName, ".")[0] + name := strings.Split(pluginConfig.FileName, ".")[0] goFile := filepath.Join(path, name+".go") soFile := filepath.Join(path, name+".so") @@ -55,13 +55,13 @@ func (c *Plugin) GetPluginPaths() (string, string, error) { } // Removes duplicate migration paths and returns them in ranked order -func (c *Plugin) GetMigrationsPaths() ([]string, error) { +func (pluginConfig *Plugin) GetMigrationsPaths() ([]string, error) { paths := make(map[uint64]string) highestRank := -1 - for name, transformer := range c.Transformers { + for name, transformer := range pluginConfig.Transformers { repo := transformer.RepositoryPath mig := transformer.MigrationPath - path := filepath.Join("$GOPATH/src", c.Home, "vendor", repo, mig) + path := filepath.Join("$GOPATH/src", pluginConfig.Home, "vendor", repo, mig) cleanPath, err := helpers.CleanPath(path) if err != nil { return nil, err @@ -96,9 +96,9 @@ func (c *Plugin) GetMigrationsPaths() ([]string, error) { } // Removes duplicate repo paths before returning them -func (c *Plugin) GetRepoPaths() map[string]bool { +func (pluginConfig *Plugin) GetRepoPaths() map[string]bool { paths := make(map[string]bool) - for _, transformer := range c.Transformers { + for _, transformer := range pluginConfig.Transformers { paths[transformer.RepositoryPath] = true } @@ -111,29 +111,29 @@ const ( UnknownTransformerType TransformerType = iota EthEvent EthStorage - EthGeneric + EthContract ) -func (pt TransformerType) String() string { +func (transformerType TransformerType) String() string { names := [...]string{ "Unknown", "eth_event", "eth_storage", - "eth_generic", + "eth_contract", } - if pt > EthGeneric || pt < EthEvent { + if transformerType > EthContract || transformerType < EthEvent { return "Unknown" } - return names[pt] + return names[transformerType] } func GetTransformerType(str string) TransformerType { types := [...]TransformerType{ EthEvent, EthStorage, - EthGeneric, + EthContract, } for _, ty := range types { diff --git a/pkg/contract_watcher/full/converter/converter.go b/pkg/contract_watcher/full/converter/converter.go index 47f3fa42..39341145 100644 --- a/pkg/contract_watcher/full/converter/converter.go +++ b/pkg/contract_watcher/full/converter/converter.go @@ -34,27 +34,21 @@ import ( // Converter is used to convert watched event logs to // custom logs containing event input name => value maps -type Converter interface { +type ConverterInterface interface { Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) Update(info *contract.Contract) } -type converter struct { +type Converter struct { ContractInfo *contract.Contract } -func NewConverter(info *contract.Contract) *converter { - return &converter{ - ContractInfo: info, - } -} - -func (c *converter) Update(info *contract.Contract) { +func (c *Converter) Update(info *contract.Contract) { c.ContractInfo = info } // Convert the given watched event log into a types.Log for the given event -func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { +func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) log := helpers.ConvertToLog(watchedEvent) diff --git a/pkg/contract_watcher/full/converter/converter_test.go b/pkg/contract_watcher/full/converter/converter_test.go index 9b7a46cb..864a53a2 100644 --- a/pkg/contract_watcher/full/converter/converter_test.go +++ b/pkg/contract_watcher/full/converter/converter_test.go @@ -39,7 +39,8 @@ var _ = Describe("Converter", func() { Describe("Update", func() { It("Updates contract con held by the converter", func() { - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) Expect(c.ContractInfo).To(Equal(con)) con := test_helpers.SetupTusdContract([]string{}, []string{}) @@ -58,7 +59,8 @@ var _ = Describe("Converter", func() { err = con.GenerateFilters() Expect(err).ToNot(HaveOccurred()) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) log, err := c.Convert(mocks.MockTranferEvent, event) Expect(err).ToNot(HaveOccurred()) @@ -77,7 +79,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["Transfer"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) _, err := c.Convert(mocks.MockTranferEvent, event) Expect(err).ToNot(HaveOccurred()) @@ -101,7 +104,8 @@ var _ = Describe("Converter", func() { It("Fails with an empty contract", func() { event := con.Events["Transfer"] - c := converter.NewConverter(&contract.Contract{}) + c := converter.Converter{} + c.Update(&contract.Contract{}) _, err = c.Convert(mocks.MockTranferEvent, event) Expect(err).To(HaveOccurred()) }) diff --git a/pkg/contract_watcher/full/transformer/transformer.go b/pkg/contract_watcher/full/transformer/transformer.go index 119aa6ec..132292ab 100644 --- a/pkg/contract_watcher/full/transformer/transformer.go +++ b/pkg/contract_watcher/full/transformer/transformer.go @@ -34,7 +34,7 @@ import ( ) // Requires a fully synced vDB and a running eth node (or infura) -type transformer struct { +type Transformer struct { // Database interfaces datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() datastore.WatchedEventRepository // Watched event log views, created by the log filters @@ -45,8 +45,8 @@ type transformer struct { retriever.BlockRetriever // Retrieves first block for contract and current block height // Processing interfaces - converter.Converter // Converts watched event logs into custom log - poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore + converter.ConverterInterface // Converts watched event logs into custom log + poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore // Store contract configuration information Config config.ContractConfig @@ -56,12 +56,12 @@ type transformer struct { } // Transformer takes in config for blockchain, database, and network id -func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *transformer { - return &transformer{ +func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer { + return &Transformer{ Poller: poller.NewPoller(BC, DB, types.FullSync), Parser: parser.NewParser(con.Network), BlockRetriever: retriever.NewBlockRetriever(DB), - Converter: converter.NewConverter(&contract.Contract{}), + ConverterInterface: &converter.Converter{}, Contracts: map[string]*contract.Contract{}, WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, FilterRepository: repositories.FilterRepository{DB: DB}, @@ -74,7 +74,7 @@ func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres. // Loops over all of the addr => filter sets // Uses parser to pull event info from abi // Use this info to generate event filters -func (tr *transformer) Init() error { +func (tr *Transformer) Init() error { for contractAddr := range tr.Config.Addresses { // Configure Abi if tr.Config.Abis[contractAddr] == "" { @@ -108,7 +108,7 @@ func (tr *transformer) Init() error { // Get contract name if it has one var name = new(string) - tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) + tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock) // Remove any potential accidental duplicate inputs in arg filter values eventArgs := map[string]bool{} @@ -162,7 +162,7 @@ func (tr *transformer) Init() error { // Uses converter to convert logs into custom log type // Persists converted logs into custuom postgres tables // Calls selected methods, using token holder address generated during event log conversion -func (tr *transformer) Execute() error { +func (tr *Transformer) Execute() error { if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts to work with") } @@ -181,7 +181,7 @@ func (tr *transformer) Execute() error { // Iterate over watched event logs for _, we := range watchedEvents { // Convert them to our custom log type - cstm, err := tr.Converter.Convert(*we, con.Events[eventSig]) + cstm, err := tr.ConverterInterface.Convert(*we, con.Events[eventSig]) if err != nil { return err } @@ -210,6 +210,6 @@ func (tr *transformer) Execute() error { return nil } -func (tr *transformer) GetConfig() config.ContractConfig { +func (tr *Transformer) GetConfig() config.ContractConfig { return tr.Config } diff --git a/pkg/contract_watcher/full/transformer/transformer_test.go b/pkg/contract_watcher/full/transformer/transformer_test.go index ee6d465e..0cb0fb31 100644 --- a/pkg/contract_watcher/full/transformer/transformer_test.go +++ b/pkg/contract_watcher/full/transformer/transformer_test.go @@ -17,307 +17,82 @@ package transformer_test import ( - "fmt" - "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "math/rand" - "strings" "time" - "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/fakes" ) var _ = Describe("Transformer", func() { - var db *postgres.DB - var err error - var blockChain core.BlockChain - var blockRepository repositories.BlockRepository - var ensAddr = strings.ToLower(constants.EnsContractAddress) - var tusdAddr = strings.ToLower(constants.TusdContractAddress) + var fakeAddress = "0x1234567890abcdef" rand.Seed(time.Now().UnixNano()) - BeforeEach(func() { - db, blockChain = test_helpers.SetupDBandBC() - blockRepository = *repositories.NewBlockRepository(db) - }) - - AfterEach(func() { - test_helpers.TearDown(db) - }) - Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Init() + blockRetriever := &fakes.MockFullBlockRetriever{} + firstBlock := int64(1) + mostRecentBlock := int64(2) + blockRetriever.FirstBlock = firstBlock + blockRetriever.MostRecentBlock = mostRecentBlock + + parsr := &fakes.MockParser{} + fakeAbi := "fake_abi" + eventName := "Transfer" + event := types.Event{} + parsr.AbiToReturn = fakeAbi + parsr.EventName = eventName + parsr.Event = event + + pollr := &fakes.MockPoller{} + fakeContractName := "fake_contract_name" + pollr.ContractName = fakeContractName + + t := getTransformer(blockRetriever, parsr, pollr) + + err := t.Init() + Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[tusdAddr] + c, ok := t.Contracts[fakeAddress] Expect(ok).To(Equal(true)) - Expect(c.StartingBlock).To(Equal(int64(6194633))) - Expect(c.LastBlock).To(Equal(int64(6194634))) - Expect(c.Abi).To(Equal(constants.TusdAbiString)) - Expect(c.Name).To(Equal("TrueUSD")) - Expect(c.Address).To(Equal(tusdAddr)) + Expect(c.StartingBlock).To(Equal(firstBlock)) + Expect(c.LastBlock).To(Equal(mostRecentBlock)) + Expect(c.Abi).To(Equal(fakeAbi)) + Expect(c.Name).To(Equal(fakeContractName)) + Expect(c.Address).To(Equal(fakeAddress)) }) It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Init() - Expect(err).To(HaveOccurred()) - }) - - It("Does nothing if watched events are unset", func() { - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - var testConf config.ContractConfig - testConf = mocks.TusdConfig - testConf.Events = nil - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).To(HaveOccurred()) - - _, ok := t.Contracts[tusdAddr] - Expect(ok).To(Equal(false)) - }) - }) - - Describe("Execute", func() { - BeforeEach(func() { - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) - blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - }) - - It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - log := test_helpers.TransferLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log) - - // We don't know vulcID, so compare individual fields instead of complete structures - Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee")) - Expect(log.Block).To(Equal(int64(6194634))) - Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21")) - Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")) - Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146")) - }) - - It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - var testConf config.ContractConfig - testConf = mocks.TusdConfig - testConf.Methods = map[string][]string{ - tusdAddr: {"balanceOf"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - c, ok := t.Contracts[tusdAddr] - Expect(ok).To(Equal(true)) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] - Expect(ok).To(Equal(false)) - - _, ok = c.EmittedAddrs[common.HexToAddress("0x")] - Expect(ok).To(Equal(false)) - - _, ok = c.EmittedAddrs[""] - Expect(ok).To(Equal(false)) - - _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] - Expect(ok).To(Equal(false)) - }) - - It("Polls given methods using generated token holder address", func() { - var testConf config.ContractConfig - testConf = mocks.TusdConfig - testConf.Methods = map[string][]string{ - tusdAddr: {"balanceOf"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - res := test_helpers.BalanceOf{} - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Balance).To(Equal("0")) - Expect(res.TokenName).To(Equal("TrueUSD")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Balance).To(Equal("0")) - Expect(res.TokenName).To(Equal("TrueUSD")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res) - Expect(err).To(HaveOccurred()) - }) - - It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - - err = t.Execute() - Expect(err).To(HaveOccurred()) - }) - }) - - Describe("Execute- against ENS registry contract", func() { - BeforeEach(func() { - blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1) - blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2) - }) - - It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) - - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - log := test_helpers.NewOwnerLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) - - // We don't know vulcID, so compare individual fields instead of complete structures - Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb")) - Expect(log.Block).To(Equal(int64(6194635))) - Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")) - Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")) - Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21")) - }) - - It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - c, ok := t.Contracts[ensAddr] - Expect(ok).To(Equal(true)) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - Expect(len(c.EmittedHashes)).To(Equal(3)) - - b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - // Doesn't keep track of address since it wouldn't be used in calling the 'owner' method - _, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] - Expect(ok).To(Equal(false)) - }) - - It("Polls given methods using generated token holder address", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - res := test_helpers.Owner{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) - Expect(res.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) - Expect(res.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res) - Expect(err).To(HaveOccurred()) - }) - - It("It does not perist events if they do not pass the emitted arg filter", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.EventArgs = map[string][]string{ - ensAddr: {"fake_filter_value"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - log := test_helpers.LightNewOwnerLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) - Expect(err).To(HaveOccurred()) - }) - - It("If a method arg filter is applied, only those arguments are used in polling", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - } - testConf.MethodArgs = map[string][]string{ - ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - res := test_helpers.Owner{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) - Expect(res.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + blockRetriever := &fakes.MockFullBlockRetriever{} + blockRetriever.FirstBlockErr = fakes.FakeError + t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) + + err := t.Init() + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) }) }) }) + +func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { + return transformer.Transformer{ + FilterRepository: &fakes.MockFilterRepository{}, + Parser: parsr, + BlockRetriever: blockRetriever, + Poller: pollr, + Contracts: map[string]*contract.Contract{}, + Config: mocks.MockConfig, + } +} diff --git a/pkg/contract_watcher/light/converter/converter.go b/pkg/contract_watcher/light/converter/converter.go index b938c61e..a1b7e051 100644 --- a/pkg/contract_watcher/light/converter/converter.go +++ b/pkg/contract_watcher/light/converter/converter.go @@ -32,28 +32,22 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" ) -type Converter interface { +type ConverterInterface interface { Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) Update(info *contract.Contract) } -type converter struct { +type Converter struct { ContractInfo *contract.Contract } -func NewConverter(info *contract.Contract) *converter { - return &converter{ - ContractInfo: info, - } -} - -func (c *converter) Update(info *contract.Contract) { +func (c *Converter) Update(info *contract.Contract) { c.ContractInfo = info } // Convert the given watched event log into a types.Log for the given event -func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { +func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) returnLogs := make([]types.Log, 0, len(logs)) for _, log := range logs { @@ -132,7 +126,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in } // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs -func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { +func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) eventsToLogs := make(map[string][]types.Log) for _, event := range events { diff --git a/pkg/contract_watcher/light/converter/converter_test.go b/pkg/contract_watcher/light/converter/converter_test.go index fb29c556..769fe057 100644 --- a/pkg/contract_watcher/light/converter/converter_test.go +++ b/pkg/contract_watcher/light/converter/converter_test.go @@ -38,7 +38,8 @@ var _ = Describe("Converter", func() { Describe("Update", func() { It("Updates contract info held by the converter", func() { con = test_helpers.SetupTusdContract(tusdWantedEvents, []string{}) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) Expect(c.ContractInfo).To(Equal(con)) info := test_helpers.SetupTusdContract([]string{}, []string{}) @@ -56,7 +57,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["Transfer"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) logs, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) Expect(len(logs)).To(Equal(2)) @@ -80,7 +82,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["Transfer"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) _, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) @@ -110,7 +113,8 @@ var _ = Describe("Converter", func() { event, ok := con.Events["NewOwner"] Expect(ok).To(Equal(true)) - c := converter.NewConverter(con) + c := converter.Converter{} + c.Update(con) _, err := c.Convert([]types.Log{mocks.MockNewOwnerLog1, mocks.MockNewOwnerLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) Expect(len(con.EmittedHashes)).To(Equal(3)) @@ -143,7 +147,8 @@ var _ = Describe("Converter", func() { It("Fails with an empty contract", func() { event := con.Events["Transfer"] - c := converter.NewConverter(&contract.Contract{}) + c := converter.Converter{} + c.Update(&contract.Contract{}) _, err = c.Convert([]types.Log{mocks.MockTransferLog1}, event, 232) Expect(err).To(HaveOccurred()) }) diff --git a/pkg/contract_watcher/light/repository/header_repository_test.go b/pkg/contract_watcher/light/repository/header_repository_test.go index 1ca2e1ac..02a94bbf 100644 --- a/pkg/contract_watcher/light/repository/header_repository_test.go +++ b/pkg/contract_watcher/light/repository/header_repository_test.go @@ -33,8 +33,8 @@ import ( var _ = Describe("Repository", func() { var db *postgres.DB var bc core.BlockChain - var omniHeaderRepo repository.HeaderRepository // omni/light header repository - var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository + var contractHeaderRepo repository.HeaderRepository // contract_watcher light header repository + var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository var eventIDs = []string{ "eventName_contractAddr", "eventName_contractAddr2", @@ -48,7 +48,7 @@ var _ = Describe("Repository", func() { BeforeEach(func() { db, bc = test_helpers.SetupDBandBC() - omniHeaderRepo = repository.NewHeaderRepository(db) + contractHeaderRepo = repository.NewHeaderRepository(db) coreHeaderRepo = repositories.NewHeaderRepository(db) }) @@ -62,7 +62,7 @@ var _ = Describe("Repository", func() { _, err := db.Exec(query) Expect(err).To(HaveOccurred()) - err = omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err = contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) _, err = db.Exec(query) @@ -70,13 +70,13 @@ var _ = Describe("Repository", func() { }) It("Caches column it creates so that it does not need to repeatedly query the database to check for it's existence", func() { - _, ok := omniHeaderRepo.CheckCache(eventIDs[0]) + _, ok := contractHeaderRepo.CheckCache(eventIDs[0]) Expect(ok).To(Equal(false)) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - v, ok := omniHeaderRepo.CheckCache(eventIDs[0]) + v, ok := contractHeaderRepo.CheckCache(eventIDs[0]) Expect(ok).To(Equal(true)) Expect(v).To(Equal(true)) }) @@ -89,7 +89,7 @@ var _ = Describe("Repository", func() { Expect(err).To(HaveOccurred()) } - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range eventIDs { @@ -100,15 +100,15 @@ var _ = Describe("Repository", func() { It("Caches columns it creates so that it does not need to repeatedly query the database to check for it's existence", func() { for _, id := range eventIDs { - _, ok := omniHeaderRepo.CheckCache(id) + _, ok := contractHeaderRepo.CheckCache(id) Expect(ok).To(Equal(false)) } - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range eventIDs { - v, ok := omniHeaderRepo.CheckCache(id) + v, ok := contractHeaderRepo.CheckCache(id) Expect(ok).To(Equal(true)) Expect(v).To(Equal(true)) } @@ -118,20 +118,20 @@ var _ = Describe("Repository", func() { Describe("MissingHeaders", func() { It("Returns all unchecked headers for the given eventID", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) }) It("Returns unchecked headers in ascending order", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) @@ -145,10 +145,10 @@ var _ = Describe("Repository", func() { It("Returns only contiguous chunks of headers", func() { addDiscontinuousHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) @@ -157,10 +157,10 @@ var _ = Describe("Repository", func() { It("Fails if eventID does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - _, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, "notEventId") + _, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, "notEventId") Expect(err).To(HaveOccurred()) }) }) @@ -168,36 +168,36 @@ var _ = Describe("Repository", func() { Describe("MissingHeadersForAll", func() { // HERE It("Returns all headers that have not been checked for all of the ids provided", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) - err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0]) + err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) + missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) - err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[1]) + err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[1]) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2]) + err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeadersForAll(6194633, 6194635, eventIDs) + missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194633, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) It("Returns only contiguous chunks of headers", func() { addDiscontinuousHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) @@ -206,10 +206,10 @@ var _ = Describe("Repository", func() { It("Returns at most 100 headers", func() { add102Headers(coreHeaderRepo, bc) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194733, eventIDs) + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194733, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(100)) Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632))) @@ -218,11 +218,11 @@ var _ = Describe("Repository", func() { It("Fails if one of the eventIDs does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) badEventIDs := append(eventIDs, "notEventId") - _, err = omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, badEventIDs) + _, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, badEventIDs) Expect(err).To(HaveOccurred()) }) }) @@ -230,36 +230,36 @@ var _ = Describe("Repository", func() { Describe("MarkHeaderChecked", func() { It("Marks the header checked for the given eventID", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id - err = omniHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0]) + err = contractHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) It("Fails if eventID does not yet exist in check_headers table", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumn(eventIDs[0]) + err := contractHeaderRepo.AddCheckColumn(eventIDs[0]) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id - err = omniHeaderRepo.MarkHeaderChecked(headerID, "notEventId") + err = contractHeaderRepo.MarkHeaderChecked(headerID, "notEventId") Expect(err).To(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) }) @@ -268,18 +268,18 @@ var _ = Describe("Repository", func() { Describe("MarkHeaderCheckedForAll", func() { It("Marks the header checked for all provided column ids", func() { addHeaders(coreHeaderRepo) - err := omniHeaderRepo.AddCheckColumns(eventIDs) + err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err := omniHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) + missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id - err = omniHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs) + err = contractHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(2)) }) @@ -296,17 +296,17 @@ var _ = Describe("Repository", func() { var missingHeaders []core.Header for _, id := range methodIDs { - err := omniHeaderRepo.AddCheckColumn(id) + err := contractHeaderRepo.AddCheckColumn(id) Expect(err).ToNot(HaveOccurred()) - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, id) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) } - err := omniHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs) + err := contractHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs) Expect(err).ToNot(HaveOccurred()) for _, id := range methodIDs { - missingHeaders, err = omniHeaderRepo.MissingHeaders(6194632, 6194635, id) + missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(0)) } @@ -317,28 +317,28 @@ var _ = Describe("Repository", func() { It("Returns headers that have been checked for all the provided events but have not been checked for all the provided methods", func() { addHeaders(coreHeaderRepo) for i, id := range eventIDs { - err := omniHeaderRepo.AddCheckColumn(id) + err := contractHeaderRepo.AddCheckColumn(id) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.AddCheckColumn(methodIDs[i]) + err = contractHeaderRepo.AddCheckColumn(methodIDs[i]) Expect(err).ToNot(HaveOccurred()) } - missingHeaders, err := omniHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) + missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0]) Expect(err).ToNot(HaveOccurred()) Expect(len(missingHeaders)).To(Equal(3)) headerID := missingHeaders[0].Id headerID2 := missingHeaders[1].Id for i, id := range eventIDs { - err = omniHeaderRepo.MarkHeaderChecked(headerID, id) + err = contractHeaderRepo.MarkHeaderChecked(headerID, id) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.MarkHeaderChecked(headerID2, id) + err = contractHeaderRepo.MarkHeaderChecked(headerID2, id) Expect(err).ToNot(HaveOccurred()) - err = omniHeaderRepo.MarkHeaderChecked(headerID, methodIDs[i]) + err = contractHeaderRepo.MarkHeaderChecked(headerID, methodIDs[i]) Expect(err).ToNot(HaveOccurred()) } - intersectionHeaders, err := omniHeaderRepo.MissingMethodsCheckedEventsIntersection(6194632, 6194635, methodIDs, eventIDs) + intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(6194632, 6194635, methodIDs, eventIDs) Expect(err).ToNot(HaveOccurred()) Expect(len(intersectionHeaders)).To(Equal(1)) Expect(intersectionHeaders[0].Id).To(Equal(headerID2)) diff --git a/pkg/contract_watcher/light/retriever/retriever_suite_test.go b/pkg/contract_watcher/light/retriever/retriever_suite_test.go index 5a254ef2..83a16269 100644 --- a/pkg/contract_watcher/light/retriever/retriever_suite_test.go +++ b/pkg/contract_watcher/light/retriever/retriever_suite_test.go @@ -27,7 +27,7 @@ import ( func TestRetriever(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Light BLock Number Retriever Suite Test") + RunSpecs(t, "Light Block Number Retriever Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/contract_watcher/light/transformer/transformer.go b/pkg/contract_watcher/light/transformer/transformer.go index 080ca040..7753e286 100644 --- a/pkg/contract_watcher/light/transformer/transformer.go +++ b/pkg/contract_watcher/light/transformer/transformer.go @@ -38,7 +38,7 @@ import ( ) // Requires a light synced vDB (headers) and a running eth node (or infura) -type transformer struct { +type Transformer struct { // Database interfaces srep.EventRepository // Holds transformed watched event log data repository.HeaderRepository // Interface for interaction with header repositories @@ -48,9 +48,9 @@ type transformer struct { retriever.BlockRetriever // Retrieves first block for contract and current block height // Processing interfaces - fetcher.Fetcher // Fetches event logs, using header hashes - converter.Converter // Converts watched event logs into custom log - poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore + fetcher.Fetcher // Fetches event logs, using header hashes + converter.ConverterInterface // Converts watched event logs into custom log + poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore // Store contract configuration information Config config.ContractConfig @@ -74,18 +74,18 @@ type transformer struct { // 4. Execute // Transformer takes in config for blockchain, database, and network id -func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *transformer { +func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer { - return &transformer{ - Poller: poller.NewPoller(bc, db, types.LightSync), - Fetcher: fetcher.NewFetcher(bc), - Parser: parser.NewParser(con.Network), - HeaderRepository: repository.NewHeaderRepository(db), - BlockRetriever: retriever.NewBlockRetriever(db), - Converter: converter.NewConverter(&contract.Contract{}), - Contracts: map[string]*contract.Contract{}, - EventRepository: srep.NewEventRepository(db, types.LightSync), - Config: con, + return &Transformer{ + Poller: poller.NewPoller(bc, db, types.LightSync), + Fetcher: fetcher.NewFetcher(bc), + Parser: parser.NewParser(con.Network), + HeaderRepository: repository.NewHeaderRepository(db), + BlockRetriever: retriever.NewBlockRetriever(db), + ConverterInterface: &converter.Converter{}, + Contracts: map[string]*contract.Contract{}, + EventRepository: srep.NewEventRepository(db, types.LightSync), + Config: con, } } @@ -93,7 +93,7 @@ func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres. // Loops over all of the addr => filter sets // Uses parser to pull event info from abi // Use this info to generate event filters -func (tr *transformer) Init() error { +func (tr *Transformer) Init() error { // Initialize internally configured transformer settings tr.contractAddresses = make([]string, 0) // Holds all contract addresses, for batch fetching of logs tr.sortedEventIds = make(map[string][]string) // Map to sort event column ids by contract, for post fetch processing and persisting of logs @@ -124,10 +124,6 @@ func (tr *transformer) Init() error { if err != nil { return err } - lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock() - if err != nil { - return err - } // Set to specified range if it falls within the bounds if firstBlock < tr.Config.StartingBlocks[contractAddr] { @@ -136,7 +132,7 @@ func (tr *transformer) Init() error { // Get contract name if it has one var name = new(string) - tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) + tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, -1) // Remove any potential accidental duplicate inputs eventArgs := map[string]bool{} @@ -201,7 +197,7 @@ func (tr *transformer) Init() error { return nil } -func (tr *transformer) Execute() error { +func (tr *Transformer) Execute() error { if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts") } @@ -253,10 +249,10 @@ func (tr *transformer) Execute() error { } // Configure converter with this contract con := tr.Contracts[conAddr] - tr.Converter.Update(con) + tr.ConverterInterface.Update(con) // Convert logs into batches of log mappings (eventName => []types.Logs - convertedLogs, err := tr.Converter.ConvertBatch(logs, con.Events, header.Id) + convertedLogs, err := tr.ConverterInterface.ConvertBatch(logs, con.Events, header.Id) if err != nil { return err } @@ -293,7 +289,7 @@ func (tr *transformer) Execute() error { } // Used to poll contract methods at a given header -func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error { +func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error { for _, con := range tr.Contracts { // Skip method polling processes if no methods are specified // Also don't try to poll methods below this contract's specified starting block @@ -317,6 +313,6 @@ func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[str return nil } -func (tr *transformer) GetConfig() config.ContractConfig { +func (tr *Transformer) GetConfig() config.ContractConfig { return tr.Config } diff --git a/pkg/contract_watcher/light/transformer/transformer_test.go b/pkg/contract_watcher/light/transformer/transformer_test.go index 9fe3d3a1..98a31722 100644 --- a/pkg/contract_watcher/light/transformer/transformer_test.go +++ b/pkg/contract_watcher/light/transformer/transformer_test.go @@ -17,425 +17,69 @@ package transformer_test import ( - "fmt" - "github.com/vulcanize/vulcanizedb/pkg/config" - "strings" - - "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" - "github.com/vulcanize/vulcanizedb/pkg/core" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" - "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/fakes" ) var _ = Describe("Transformer", func() { - var db *postgres.DB - var err error - var blockChain core.BlockChain - var headerRepository repositories.HeaderRepository - var headerID int64 - var ensAddr = strings.ToLower(constants.EnsContractAddress) - var tusdAddr = strings.ToLower(constants.TusdContractAddress) - - BeforeEach(func() { - db, blockChain = test_helpers.SetupDBandBC() - headerRepository = repositories.NewHeaderRepository(db) - }) - - AfterEach(func() { - test_helpers.TearDown(db) - }) - + var fakeAddress = "0x1234567890abcdef" Describe("Init", func() { It("Initializes transformer's contract objects", func() { - headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) - headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Init() + blockRetriever := &fakes.MockLightBlockRetriever{} + firstBlock := int64(1) + blockRetriever.FirstBlock = firstBlock + + parsr := &fakes.MockParser{} + fakeAbi := "fake_abi" + parsr.AbiToReturn = fakeAbi + + pollr := &fakes.MockPoller{} + fakeContractName := "fake_contract_name" + pollr.ContractName = fakeContractName + + t := getFakeTransformer(blockRetriever, parsr, pollr) + + err := t.Init() + Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[tusdAddr] + c, ok := t.Contracts[fakeAddress] Expect(ok).To(Equal(true)) - Expect(c.StartingBlock).To(Equal(int64(6194632))) + Expect(c.StartingBlock).To(Equal(firstBlock)) Expect(c.LastBlock).To(Equal(int64(-1))) - Expect(c.Abi).To(Equal(constants.TusdAbiString)) - Expect(c.Name).To(Equal("TrueUSD")) - Expect(c.Address).To(Equal(tusdAddr)) + Expect(c.Abi).To(Equal(fakeAbi)) + Expect(c.Name).To(Equal(fakeContractName)) + Expect(c.Address).To(Equal(fakeAddress)) }) - It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Init() - Expect(err).To(HaveOccurred()) - }) - - It("Does nothing if nothing if no addresses are configured", func() { - headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) - headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) - var testConf config.ContractConfig - testConf = mocks.TusdConfig - testConf.Addresses = nil - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - - _, ok := t.Contracts[tusdAddr] - Expect(ok).To(Equal(false)) - }) - }) - - Describe("Execute- against TrueUSD contract", func() { - BeforeEach(func() { - header1, err := blockChain.GetHeaderByNumber(6791668) - Expect(err).ToNot(HaveOccurred()) - header2, err := blockChain.GetHeaderByNumber(6791669) - Expect(err).ToNot(HaveOccurred()) - header3, err := blockChain.GetHeaderByNumber(6791670) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header1) - headerID, err = headerRepository.CreateOrUpdateHeader(header2) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header3) - }) - - It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - log := test_helpers.LightTransferLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log) - Expect(err).ToNot(HaveOccurred()) - // We don't know vulcID, so compare individual fields instead of complete structures - Expect(log.HeaderID).To(Equal(headerID)) - Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) - Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) - Expect(log.Value).To(Equal("9998940000000000000000")) - }) - - It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - var testConf config.ContractConfig - testConf = mocks.TusdConfig - testConf.Methods = map[string][]string{ - tusdAddr: {"balanceOf"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[tusdAddr] - Expect(ok).To(Equal(true)) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - Expect(len(c.EmittedAddrs)).To(Equal(4)) - Expect(len(c.EmittedHashes)).To(Equal(0)) - - b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] - Expect(ok).To(Equal(false)) - - _, ok = c.EmittedAddrs[common.HexToAddress("0x")] - Expect(ok).To(Equal(false)) - - _, ok = c.EmittedAddrs[""] - Expect(ok).To(Equal(false)) - - _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] - Expect(ok).To(Equal(false)) - }) - - It("Polls given methods using generated token holder address", func() { - var testConf config.ContractConfig - testConf = mocks.TusdConfig - testConf.Methods = map[string][]string{ - tusdAddr: {"balanceOf"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - res := test_helpers.BalanceOf{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Balance).To(Equal("55849938025000000000000")) - Expect(res.TokenName).To(Equal("TrueUSD")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res) - Expect(err).To(HaveOccurred()) - }) - - It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) - err = t.Execute() - Expect(err).To(HaveOccurred()) - }) - }) - - Describe("Execute- against ENS registry contract", func() { - BeforeEach(func() { - header1, err := blockChain.GetHeaderByNumber(6885695) - Expect(err).ToNot(HaveOccurred()) - header2, err := blockChain.GetHeaderByNumber(6885696) - Expect(err).ToNot(HaveOccurred()) - header3, err := blockChain.GetHeaderByNumber(6885697) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header1) - headerID, err = headerRepository.CreateOrUpdateHeader(header2) - Expect(err).ToNot(HaveOccurred()) - headerRepository.CreateOrUpdateHeader(header3) - }) - - It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - log := test_helpers.LightNewOwnerLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) - Expect(err).ToNot(HaveOccurred()) - // We don't know vulcID, so compare individual fields instead of complete structures - Expect(log.HeaderID).To(Equal(headerID)) - Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) - Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) - Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) - }) - - It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[ensAddr] - Expect(ok).To(Equal(true)) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - Expect(len(c.EmittedHashes)).To(Equal(2)) - Expect(len(c.EmittedAddrs)).To(Equal(0)) - - b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - // Doesn't keep track of address since it wouldn't be used in calling the 'owner' method - _, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] - Expect(ok).To(Equal(false)) - }) - - It("Polls given method using list of collected hashes", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - res := test_helpers.Owner{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) - Expect(res.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) - Expect(res.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res) - Expect(err).To(HaveOccurred()) - }) - - It("It does not persist events if they do not pass the emitted arg filter", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.EventArgs = map[string][]string{ - ensAddr: {"fake_filter_value"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - log := test_helpers.LightNewOwnerLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) - Expect(err).To(HaveOccurred()) - }) - - It("If a method arg filter is applied, only those arguments are used in polling", func() { - var testConf config.ContractConfig - testConf = mocks.ENSConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - } - testConf.MethodArgs = map[string][]string{ - ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - res := test_helpers.Owner{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) - Expect(res.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) - Expect(err).To(HaveOccurred()) - }) - }) - - Describe("Execute- against both ENS and TrueUSD", func() { - BeforeEach(func() { - for i := 6885692; i < 6885702; i++ { - header, err := blockChain.GetHeaderByNumber(int64(i)) - Expect(err).ToNot(HaveOccurred()) - _, err = headerRepository.CreateOrUpdateHeader(header) - Expect(err).ToNot(HaveOccurred()) - } - }) - - It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - newOwnerLog := test_helpers.LightNewOwnerLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) - Expect(err).ToNot(HaveOccurred()) - // We don't know vulcID, so compare individual fields instead of complete structures - Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) - Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) - Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) - - transferLog := test_helpers.LightTransferLog{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog) - Expect(err).ToNot(HaveOccurred()) - // We don't know vulcID, so compare individual fields instead of complete structures - Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")) - Expect(transferLog.To).To(Equal("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")) - Expect(transferLog.Value).To(Equal("2800000000000000000000")) - }) - - It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { - var testConf config.ContractConfig - testConf = mocks.ENSandTusdConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - tusdAddr: {"balanceOf"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - ens, ok := t.Contracts[ensAddr] - Expect(ok).To(Equal(true)) - tusd, ok := t.Contracts[tusdAddr] - Expect(ok).To(Equal(true)) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - Expect(len(ens.EmittedHashes)).To(Equal(2)) - Expect(len(ens.EmittedAddrs)).To(Equal(0)) - Expect(len(tusd.EmittedAddrs)).To(Equal(2)) - Expect(len(tusd.EmittedHashes)).To(Equal(0)) - - b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = tusd.EmittedAddrs[common.HexToAddress("0x8cA465764873E71CEa525F5EB6AE973d650c22C2")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - b, ok = tusd.EmittedAddrs[common.HexToAddress("0xc338482360651E5D30BEd77b7c85358cbBFB2E0e")] - Expect(ok).To(Equal(true)) - Expect(b).To(Equal(true)) - - _, ok = tusd.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] - Expect(ok).To(Equal(false)) - }) - - It("Polls given methods for each contract, using list of collected values", func() { - var testConf config.ContractConfig - testConf = mocks.ENSandTusdConfig - testConf.Methods = map[string][]string{ - ensAddr: {"owner"}, - tusdAddr: {"balanceOf"}, - } - t := transformer.NewTransformer(testConf, blockChain, db) - err = t.Init() - Expect(err).ToNot(HaveOccurred()) - err = t.Execute() - Expect(err).ToNot(HaveOccurred()) - - owner := test_helpers.Owner{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner) - Expect(err).ToNot(HaveOccurred()) - Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) - Expect(owner.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner) - Expect(err).ToNot(HaveOccurred()) - Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000")) - Expect(owner.TokenName).To(Equal("")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner) - Expect(err).To(HaveOccurred()) - - bal := test_helpers.BalanceOf{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal) - Expect(err).ToNot(HaveOccurred()) - Expect(bal.Balance).To(Equal("1954436000000000000000")) - Expect(bal.TokenName).To(Equal("TrueUSD")) - - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal) + It("Fails to initialize if first block cannot be fetched from vDB headers table", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + blockRetriever.FirstBlockErr = fakes.FakeError + t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) + + err := t.Init() + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) }) }) }) + +func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { + return transformer.Transformer{ + Parser: parsr, + BlockRetriever: blockRetriever, + Poller: pollr, + HeaderRepository: &fakes.MockLightHeaderRepository{}, + Contracts: map[string]*contract.Contract{}, + Config: mocks.MockConfig, + } +} diff --git a/pkg/contract_watcher/shared/contract/contract.go b/pkg/contract_watcher/shared/contract/contract.go index d6f62d0a..b3eb3b7f 100644 --- a/pkg/contract_watcher/shared/contract/contract.go +++ b/pkg/contract_watcher/shared/contract/contract.go @@ -67,7 +67,7 @@ func (c Contract) Init() *Contract { return &c } -// Use contract info to generate event filters - full sync omni watcher only +// Use contract info to generate event filters - full sync contract watcher only func (c *Contract) GenerateFilters() error { c.Filters = map[string]filters.LogFilter{} diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go index e65db708..f9224847 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go @@ -18,13 +18,13 @@ package mocks import ( "encoding/json" - "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" @@ -262,74 +262,46 @@ var MockNewOwnerLog2 = types.Log{ var ens = strings.ToLower(constants.EnsContractAddress) var tusd = strings.ToLower(constants.TusdContractAddress) -var TusdConfig = config.ContractConfig{ +var MockConfig = config.ContractConfig{ Network: "", Addresses: map[string]bool{ - tusd: true, + "0x1234567890abcdef": true, }, Abis: map[string]string{ - tusd: "", + "0x1234567890abcdef": "fake_abi", }, Events: map[string][]string{ - tusd: []string{"Transfer"}, + "0x1234567890abcdef": []string{"Transfer"}, }, Methods: map[string][]string{ - tusd: nil, + "0x1234567890abcdef": nil, }, MethodArgs: map[string][]string{ - tusd: nil, + "0x1234567890abcdef": nil, }, EventArgs: map[string][]string{ - tusd: nil, + "0x1234567890abcdef": nil, }, } -var ENSConfig = config.ContractConfig{ +var MockEmptyConfig = config.ContractConfig{ Network: "", Addresses: map[string]bool{ - ens: true, + "0x1234567890abcdef": true, }, Abis: map[string]string{ - ens: "", + "0x1234567890abcdef": "fake_abi", }, Events: map[string][]string{ - ens: []string{"NewOwner"}, + "0x1234567890abcdef": nil, }, Methods: map[string][]string{ - ens: nil, + "0x1234567890abcdef": nil, }, MethodArgs: map[string][]string{ - ens: nil, + "0x1234567890abcdef": nil, }, EventArgs: map[string][]string{ - ens: nil, - }, -} - -var ENSandTusdConfig = config.ContractConfig{ - Network: "", - Addresses: map[string]bool{ - ens: true, - tusd: true, - }, - Abis: map[string]string{ - ens: "", - tusd: "", - }, - Events: map[string][]string{ - ens: []string{"NewOwner"}, - tusd: []string{"Transfer"}, - }, - Methods: map[string][]string{ - ens: nil, - tusd: nil, - }, - MethodArgs: map[string][]string{ - ens: nil, - tusd: nil, - }, - EventArgs: map[string][]string{ - ens: nil, - tusd: nil, + "0x1234567890abcdef": nil, }, } diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/test_data.go b/pkg/contract_watcher/shared/helpers/test_helpers/test_data.go new file mode 100644 index 00000000..c52fd20c --- /dev/null +++ b/pkg/contract_watcher/shared/helpers/test_helpers/test_data.go @@ -0,0 +1,109 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package test_helpers + +import ( + "strings" + + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants" +) + +var ens = strings.ToLower(constants.EnsContractAddress) +var tusd = strings.ToLower(constants.TusdContractAddress) + +var TusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + tusd: true, + }, + Abis: map[string]string{ + tusd: "", + }, + Events: map[string][]string{ + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + tusd: nil, + }, + MethodArgs: map[string][]string{ + tusd: nil, + }, + EventArgs: map[string][]string{ + tusd: nil, + }, + StartingBlocks: map[string]int64{ + tusd: 5197514, + }, +} + +var ENSConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + }, + Abis: map[string]string{ + ens: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + }, + Methods: map[string][]string{ + ens: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + }, + StartingBlocks: map[string]int64{ + ens: 3327417, + }, +} + +var ENSandTusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + tusd: true, + }, + Abis: map[string]string{ + ens: "", + tusd: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + ens: nil, + tusd: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, + StartingBlocks: map[string]int64{ + ens: 3327417, + tusd: 5197514, + }, +} diff --git a/pkg/contract_watcher/shared/repository/event_repository_test.go b/pkg/contract_watcher/shared/repository/event_repository_test.go index d6c3bff9..44a4cd66 100644 --- a/pkg/contract_watcher/shared/repository/event_repository_test.go +++ b/pkg/contract_watcher/shared/repository/event_repository_test.go @@ -134,7 +134,8 @@ var _ = Describe("Repository", func() { Describe("PersistLogs", func() { BeforeEach(func() { - c := fc.NewConverter(con) + c := fc.Converter{} + c.Update(con) log, err = c.Convert(mockEvent, event) Expect(err).ToNot(HaveOccurred()) }) @@ -276,7 +277,8 @@ var _ = Describe("Repository", func() { headerRepository := repositories.NewHeaderRepository(db) headerID, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) Expect(err).ToNot(HaveOccurred()) - c := lc.NewConverter(con) + c := lc.Converter{} + c.Update(con) logs, err = c.Convert([]geth.Log{mockLog1, mockLog2}, event, headerID) Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/contract_watcher/shared/retriever/address_retriever_test.go b/pkg/contract_watcher/shared/retriever/address_retriever_test.go index 7ff4ced7..c8aa6fff 100644 --- a/pkg/contract_watcher/shared/retriever/address_retriever_test.go +++ b/pkg/contract_watcher/shared/retriever/address_retriever_test.go @@ -64,7 +64,8 @@ var _ = Describe("Address Retriever Test", func() { err = info.GenerateFilters() Expect(err).ToNot(HaveOccurred()) - c := converter.NewConverter(info) + c := converter.Converter{} + c.Update(info) log, err = c.Convert(mockEvent, event) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/contract_watcher/shared/types/event.go b/pkg/contract_watcher/shared/types/event.go index 7033f4ed..2818cc8d 100644 --- a/pkg/contract_watcher/shared/types/event.go +++ b/pkg/contract_watcher/shared/types/event.go @@ -38,7 +38,7 @@ type Field struct { // Struct to hold instance of an event log data type Log struct { - Id int64 // VulcanizeIdLog for full sync and header ID for light sync omni watcher + Id int64 // VulcanizeIdLog for full sync and header ID for light sync contract watcher Values map[string]string // Map of event input names to their values // Used for full sync only diff --git a/pkg/contract_watcher/shared/types/mode.go b/pkg/contract_watcher/shared/types/mode.go index 62057a16..9f37a857 100644 --- a/pkg/contract_watcher/shared/types/mode.go +++ b/pkg/contract_watcher/shared/types/mode.go @@ -47,7 +47,7 @@ func (mode Mode) MarshalText() ([]byte, error) { case FullSync: return []byte("full"), nil default: - return nil, fmt.Errorf("omni watcher: unknown mode %d, want LightSync or FullSync", mode) + return nil, fmt.Errorf("contract watcher: unknown mode %d, want LightSync or FullSync", mode) } } @@ -58,7 +58,7 @@ func (mode *Mode) UnmarshalText(text []byte) error { case "full": *mode = FullSync default: - return fmt.Errorf(`omni watcher: unknown mode %q, want "light" or "full"`, text) + return fmt.Errorf(`contract watcher: unknown mode %q, want "light" or "full"`, text) } return nil } diff --git a/pkg/fakes/mock_parser.go b/pkg/fakes/mock_parser.go index e705f413..dd03bbab 100644 --- a/pkg/fakes/mock_parser.go +++ b/pkg/fakes/mock_parser.go @@ -15,8 +15,9 @@ func (*MockParser) Parse(contractAddr string) error { return nil } -func (*MockParser) ParseAbiStr(abiStr string) error { - panic("implement me") +func (m *MockParser) ParseAbiStr(abiStr string) error { + m.AbiToReturn = abiStr + return nil } func (parser *MockParser) Abi() string { diff --git a/pkg/plugin/writer/writer.go b/pkg/plugin/writer/writer.go index 33e8a84c..e888bc5b 100644 --- a/pkg/plugin/writer/writer.go +++ b/pkg/plugin/writer/writer.go @@ -74,7 +74,7 @@ func (w *writer) WritePlugin() error { f.Func().Params(Id("e").Id("exporter")).Id("Export").Params().Parens(List( Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "EventTransformerInitializer"), Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "StorageTransformerInitializer"), - Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "GenericTransformerInitializer"), + Index().Qual("github.com/vulcanize/vulcanizedb/libraries/shared/transformer", "ContractTransformerInitializer"), )).Block(Return( Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", @@ -84,7 +84,7 @@ func (w *writer) WritePlugin() error { "StorageTransformerInitializer").Values(code[config.EthStorage]...), Index().Qual( "github.com/vulcanize/vulcanizedb/libraries/shared/transformer", - "GenericTransformerInitializer").Values(code[config.EthGeneric]...))) // Exports the collected event and storage transformer initializers + "ContractTransformerInitializer").Values(code[config.EthContract]...))) // Exports the collected event and storage transformer initializers // Write code to destination file err = f.Save(goFile) @@ -104,8 +104,8 @@ func (w *writer) collectTransformers() (map[config.TransformerType][]Code, error code[config.EthEvent] = append(code[config.EthEvent], Qual(path, "EventTransformerInitializer")) case config.EthStorage: code[config.EthStorage] = append(code[config.EthStorage], Qual(path, "StorageTransformerInitializer")) - case config.EthGeneric: - code[config.EthGeneric] = append(code[config.EthGeneric], Qual(path, "GenericTransformerInitializer")) + case config.EthContract: + code[config.EthContract] = append(code[config.EthContract], Qual(path, "ContractTransformerInitializer")) default: return nil, errors.New(fmt.Sprintf("invalid transformer type %s", transformer.Type)) } From d3e172d9ab685585fbbe8e5599fc682a3ffcae88 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 14 Mar 2019 15:07:39 -0500 Subject: [PATCH 09/10] remove references to infura token except for in infura.toml so that tests will continue to pass on CI --- README.md | 16 +++++++++++++--- cmd/contractWatcher.go | 2 +- environments/example.toml | 4 ++-- environments/infura.toml | 8 ++++---- environments/private.toml | 8 ++++---- environments/public.toml.example | 10 +++++----- .../shared/getter/getter_test.go | 5 +++-- .../shared/helpers/test_helpers/database.go | 18 +++--------------- 8 files changed, 35 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index e9787b6c..381db3e1 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ This command takes a config of the form: port = 5432 [client] - ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + ipcPath = "path_to_ethjson_rpc" [contract] network = "" @@ -248,9 +248,19 @@ The 'method' and 'event' identifiers are tacked onto the end of the table names Example: -Running `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light` +Modify `./environments/example.toml` to replace `"path_to_ethjson_rpc"` with a path that points to an ethjson_rpc endpoint (e.g. a local geth node ipc path or an Infura url). +This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node, +remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract. -Runs our contract watcher in light mode, configured to watch the contracts specified in the config file. Note that +If you are operating a light sync vDB, run: + + `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light` + +If instead you are operating a full sync vDB and provided an archival node IPC path, run in full mode: + + `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=full` + +This will run the contractWatcher and configures it to watch the contracts specified in the config file. Note that by default we operate in `light` mode but the flag is included here to demonstrate its use. The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E). diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index 9305249f..cfdfd7d9 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -46,7 +46,7 @@ Requires a .toml config file: port = 5432 [client] - ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + ipcPath = "path_to_ethjson_rpc" [contract] network = "" diff --git a/environments/example.toml b/environments/example.toml index 66522f05..9d253390 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -1,10 +1,10 @@ [database] - name = "vulcanize_infura" + name = "vulcanize_public" hostname = "localhost" port = 5432 [client] - ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + ipcPath = "path_to_ethjson_rpc" [contract] network = "" diff --git a/environments/infura.toml b/environments/infura.toml index 095e747d..d9afd385 100644 --- a/environments/infura.toml +++ b/environments/infura.toml @@ -1,7 +1,7 @@ [database] -name = "vulcanize_public" -hostname = "localhost" -port = 5432 + name = "vulcanize_public" + hostname = "localhost" + port = 5432 [client] -ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" \ No newline at end of file + ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" \ No newline at end of file diff --git a/environments/private.toml b/environments/private.toml index 30382733..83f80f7b 100644 --- a/environments/private.toml +++ b/environments/private.toml @@ -1,7 +1,7 @@ [database] -name = "vulcanize_private" -hostname = "localhost" -port = 5432 + name = "vulcanize_private" + hostname = "localhost" + port = 5432 [client] -ipcPath = "http://127.0.0.1:7545" \ No newline at end of file + ipcPath = "http://127.0.0.1:7545" diff --git a/environments/public.toml.example b/environments/public.toml.example index 1e748bfb..52f8a2d1 100644 --- a/environments/public.toml.example +++ b/environments/public.toml.example @@ -1,8 +1,8 @@ [database] -name = "vulcanize_public" -hostname = "localhost" -port = 5432 + name = "vulcanize_public" + hostname = "localhost" + port = 5432 [client] -ipcPath = -levelDbPath = \ No newline at end of file + ipcPath = + levelDbPath = diff --git a/pkg/contract_watcher/shared/getter/getter_test.go b/pkg/contract_watcher/shared/getter/getter_test.go index c84560ce..0f5d7cb6 100644 --- a/pkg/contract_watcher/shared/getter/getter_test.go +++ b/pkg/contract_watcher/shared/getter/getter_test.go @@ -28,15 +28,16 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" + "github.com/vulcanize/vulcanizedb/test_config" ) var _ = Describe("Interface Getter", func() { Describe("GetAbi", func() { It("Constructs and returns a custom abi based on results from supportsInterface calls", func() { expectedABI := `[` + constants.AddrChangeInterface + `,` + constants.NameChangeInterface + `,` + constants.ContentChangeInterface + `,` + constants.AbiChangeInterface + `,` + constants.PubkeyChangeInterface + `]` - + con := test_config.InfuraClient + infuraIPC := con.IPCPath blockNumber := int64(6885696) - infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" rawRpcClient, err := rpc.Dial(infuraIPC) Expect(err).NotTo(HaveOccurred()) rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index 69ed26a3..7c297bef 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -34,6 +34,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/geth/client" rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" "github.com/vulcanize/vulcanizedb/pkg/geth/node" + "github.com/vulcanize/vulcanizedb/test_config" ) type TransferLog struct { @@ -106,22 +107,9 @@ type Owner struct { Address string `db:"returned"` } -func SetupBC() core.BlockChain { - infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" - rawRpcClient, err := rpc.Dial(infuraIPC) - Expect(err).NotTo(HaveOccurred()) - rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) - ethClient := ethclient.NewClient(rawRpcClient) - blockChainClient := client.NewEthClient(ethClient) - node := node.MakeNode(rpcClient) - transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) - - return blockChain -} - func SetupDBandBC() (*postgres.DB, core.BlockChain) { - infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" + con := test_config.InfuraClient + infuraIPC := con.IPCPath rawRpcClient, err := rpc.Dial(infuraIPC) Expect(err).NotTo(HaveOccurred()) rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) From 1aa849bcb4650656867f9d4bd61371bc02c3df5a Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Thu, 14 Mar 2019 16:49:27 -0500 Subject: [PATCH 10/10] PR fixes; remove all infura token references and setup travis to use encrypted env variable; rest of the ethjson_rpc dependent tests extracted to integration_test --- .travis.yml | 11 +-- README.md | 27 +++---- cmd/compose.go | 2 +- cmd/composeAndExecute.go | 4 +- cmd/contractWatcher.go | 8 +- cmd/execute.go | 21 ++---- environments/example.toml | 2 +- environments/infura.toml | 2 +- .../contract_watcher_full_transformer_test.go | 1 - ...contract_watcher_light_transformer_test.go | 5 +- .../poller_test.go | 35 ++++----- libraries/shared/factories/event/README.md | 2 +- libraries/shared/watcher/contract_watcher.go | 3 +- pkg/config/contract.go | 57 ++++++++------- .../full/converter/converter.go | 4 +- .../full/transformer/transformer.go | 73 +++++++++++-------- .../full/transformer/transformer_test.go | 6 +- .../light/converter/converter.go | 4 +- .../light/repository/header_repository.go | 6 +- .../repository/header_repository_test.go | 13 ++-- .../light/transformer/transformer.go | 70 +++++++++--------- .../light/transformer/transformer_test.go | 45 +++++++++++- .../shared/contract/contract.go | 1 - .../shared/helpers/test_helpers/database.go | 4 +- pkg/contract_watcher/shared/parser/parser.go | 2 +- pkg/contract_watcher/shared/poller/poller.go | 6 +- .../shared/poller/poller_suite_test.go | 35 --------- .../shared/repository/event_repository.go | 4 +- .../shared/repository/method_repository.go | 2 +- pkg/fakes/mock_poller.go | 2 +- pkg/plugin/test_helpers/database.go | 2 +- test_config/test_config.go | 15 +++- 32 files changed, 242 insertions(+), 232 deletions(-) rename {pkg/contract_watcher/shared/poller => integration_test}/poller_test.go (92%) delete mode 100644 pkg/contract_watcher/shared/poller/poller_suite_test.go diff --git a/.travis.yml b/.travis.yml index 237dc152..1faa79d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,28 +5,25 @@ go: services: - postgresql addons: - postgresql: "9.6" - + postgresql: '9.6' go_import_path: github.com/vulcanize/vulcanizedb - before_install: - # ginkgo golint dep goose - make installtools - bash ./scripts/install-postgres-10.sh - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list - sudo apt-get update && sudo apt-get install yarn - before_script: - sudo -u postgres createdb vulcanize_private - make migrate NAME=vulcanize_private - cd postgraphile && yarn - script: - yarn test - cd ../ - make test - make integrationtest - notifications: email: false +env: + matrix: + secure: hz6YPkTm59QhOQ2+05K+AWHw0wOoPjz9wqfUKZcuUi+ICdcuClXMt+hVpUDmwrGOCp8B5y8hyxr/iq7P+3MLwz1/2JYxc9S9MT1O0WXU8ZuWZR0LI1e2ZhZCF3E/JWq6c4atxFIEhcv7roBUkbUcRA8cpRdZmEmpSM8JyP0z76CezIG/HeyUdVW34DLjTBNB0TJdOZyfOh0aCvzgXR/kfKaSYNBhJY7j2UK7x0qK0UlQ/n7RHCrtjWoNWpuwl9bw1F5plMHOD9bq0oDG6gs1SFBaybfEMN71Hp0QxhD/u+1tVuHfGooYhzVgxStPSCpSkgQ7vgSZI766ErqPc3B6Wv9K+s5exPLgCykEiLorW6qI8A+mdiPIiIzLBRMcbF/kCo7gFh0kDIYbSTjS5COfjNw/fKsp59upXF4VtCDgVgjAemY6XT4lziZiVQwiK1Oyln8HrIux1aJEWgRGEQpQqwVeCUHClHus5Paf/N0Ci5f9NHh2zbkZvDuUF2uQu6Wc58wXHcVsloyfQibJrH1q2sQRqdiPfN5Y9l0igFzXILFd4itXMyMnSDQh6+GD6V0YY/hGufAs42UymjGYbEwZQP/gn8/bQpilngbmlJ7bB2nva70kXgqhuNiZM5XuFuQMIP3U2Tbm87FHEFUCoKPv2if9Ft74YkAuzVljAMo2YLQ= diff --git a/README.md b/README.md index 381db3e1..91e5cd99 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ false If you have full rinkeby chaindata you can move it to `rinkeby_vulcanizedb_geth_data` docker volume to skip long wait of sync. ## Running the Tests +- Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full archival node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url) - `createdb vulcanize_private` will create the test db - `make migrate NAME=vulcanize_private` will run the db migrations - `make test` will run the unit tests and skip the integration tests @@ -154,12 +155,12 @@ Contract watchers work with a light or full sync vDB to fetch raw ethereum data A watcher is composed of at least a fetcher and a transformer or set of transformers, where a fetcher is an interface for retrieving raw Ethereum data from some source (e.g. eth_jsonrpc, IPFS) and a transformer is an interface for filtering through that raw Ethereum data to extract, process, and persist data for specific contracts or accounts. -### contractWatcher +## contractWatcher The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. It also provides some state variable coverage by automating polling of public methods, with some restrictions: 1. The method must have 2 or less arguments -2. The method's arguments must all be of type address or bytes32 (hash) -3. The method must return a single value +1. The method's arguments must all be of type address or bytes32 (hash) +1. The method must return a single value This command operates in two modes- `light` and `full`- which require a light or full-synced vulcanizeDB, respectively. @@ -177,7 +178,7 @@ This command takes a config of the form: port = 5432 [client] - ipcPath = "path_to_ethjson_rpc" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [contract] network = "" @@ -238,7 +239,7 @@ This command takes a config of the form: At the very minimum, for each contract address an ABI and a starting block number need to be provided (or just the starting block if the ABI can be reliably fetched from Etherscan). With just this information we will be able to watch all events at the contract, but with no additional filters and no method polling. -#### contractWatcher output +### contractWatcher output Transformed events and polled method results are committed to Postgres in schemas and tables generated according to the contract abi. @@ -246,9 +247,9 @@ Schemas are created for each contract using the naming convention `_< Under this schema, tables are generated for watched events as `_event` and for polled methods as `_method` The 'method' and 'event' identifiers are tacked onto the end of the table names to prevent collisions between methods and events of the same lowercase name -Example: +### contractWatcher example: -Modify `./environments/example.toml` to replace `"path_to_ethjson_rpc"` with a path that points to an ethjson_rpc endpoint (e.g. a local geth node ipc path or an Infura url). +Modify `./environments/example.toml` to replace the empty `ipcPath` with a path that points to an ethjson_rpc endpoint (e.g. a local geth node ipc path or an Infura url). This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node, remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract. @@ -314,14 +315,14 @@ The addition of '_' after table names is to prevent collisions with reserved Pos Also notice that the contract address used for the schema name has been down-cased. -### composeAndExecute +## composeAndExecute The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers. This is accomplished by generating a Go pluggin which allows our `vulcanizedb` binary to link to external transformers, so long as they abide by our standard [interfaces](https://github.com/vulcanize/maker-vulcanizedb/tree/compose_and_execute/libraries/shared/transformer). This command requires Go 1.11+ and [Go plugins](https://golang.org/pkg/plugin/) only work on Unix based systems. -#### Writing custom transformers +### Writing custom transformers Storage Transformers * [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/README.md) * [Example](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/EXAMPLE.md) @@ -330,7 +331,7 @@ Event Transformers * [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/event_docs/libraries/shared/factories/README.md) * [Example](https://github.com/vulcanize/ens_transformers/tree/working) -#### composeAndExecute configuration +### composeAndExecute configuration A .toml config file is specified when executing the command: `./vulcanizedb composeAndExecute --config=./environments/config_name.toml` @@ -345,7 +346,7 @@ The config provides information for composing a set of transformers: port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] home = "github.com/vulcanize/vulcanizedb" @@ -365,7 +366,7 @@ The config provides information for composing a set of transformers: rank = "0" [exporter.transformer2] path = "path/to/transformer2" - type = "eth_generic" + type = "eth_contract" repository = "github.com/account/repo" migrations = "db/migrations" rank = "0" @@ -441,7 +442,7 @@ func (e exporter) Export() []interface1.EventTransformerInitializer, []interface } ``` -#### Preparing transformer(s) to work as pluggins for composeAndExecute +### Preparing transformers to work as pluggins for composeAndExecute To plug in an external transformer we need to: * Create a [package](https://github.com/vulcanize/ens_transformers/blob/working/transformers/registry/new_owner/initializer/initializer.go) diff --git a/cmd/compose.go b/cmd/compose.go index 28de4af1..b8098b9b 100644 --- a/cmd/compose.go +++ b/cmd/compose.go @@ -42,7 +42,7 @@ var composeCmd = &cobra.Command{ port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] home = "github.com/vulcanize/vulcanizedb" diff --git a/cmd/composeAndExecute.go b/cmd/composeAndExecute.go index c6aa318b..d2abd045 100644 --- a/cmd/composeAndExecute.go +++ b/cmd/composeAndExecute.go @@ -42,7 +42,7 @@ var composeAndExecuteCmd = &cobra.Command{ port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] home = "github.com/vulcanize/vulcanizedb" @@ -180,7 +180,7 @@ func composeAndExecute() { gw := watcher.NewContractWatcher(&db, blockChain) gw.AddTransformers(ethContractInitializers) wg.Add(1) - go contractWatching(&gw, &wg) + go watchEthContract(&gw, &wg) } wg.Wait() } diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go index cfdfd7d9..d27802ce 100644 --- a/cmd/contractWatcher.go +++ b/cmd/contractWatcher.go @@ -46,7 +46,7 @@ Requires a .toml config file: port = 5432 [client] - ipcPath = "path_to_ethjson_rpc" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [contract] network = "" @@ -112,12 +112,14 @@ func contractWatcher() { } for range ticker.C { - t.Execute() + err = t.Execute() + if err != nil { + log.Error("Execution error for transformer:", t.GetConfig().Name, err) + } } } func init() { rootCmd.AddCommand(contractWatcherCmd) - contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") } diff --git a/cmd/execute.go b/cmd/execute.go index 3010db59..3ebc15f1 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -46,7 +46,7 @@ var executeCmd = &cobra.Command{ port = 5432 [client] - ipcPath = "http://kovan0.vulcanize.io:8545" + ipcPath = "/Users/user/Library/Ethereum/geth.ipc" [exporter] name = "exampleTransformerExporter" @@ -128,7 +128,7 @@ func execute() { gw := watcher.NewContractWatcher(&db, blockChain) gw.AddTransformers(ethContractInitializers) wg.Add(1) - go contractWatching(&gw, &wg) + go watchEthContract(&gw, &wg) } wg.Wait() } @@ -155,10 +155,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { - err := w.Execute(recheck) - if err != nil { - // TODO Handle watcher errors in execute - } + w.Execute(recheck) } } @@ -169,23 +166,17 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) { ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { - err := w.Execute() - if err != nil { - // TODO Handle watcher errors in execute - } + w.Execute() } } -func contractWatching(w *watcher.ContractWatcher, wg *syn.WaitGroup) { +func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) { defer wg.Done() // Execute over the ContractTransformerInitializer set using the contract watcher log.Info("executing contract_watcher transformers") ticker := time.NewTicker(pollingInterval) defer ticker.Stop() for range ticker.C { - err := w.Execute(nil) - if err != nil { - // TODO Handle watcher errors in execute - } + w.Execute() } } diff --git a/environments/example.toml b/environments/example.toml index 9d253390..e025baaa 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -4,7 +4,7 @@ port = 5432 [client] - ipcPath = "path_to_ethjson_rpc" + ipcPath = "" [contract] network = "" diff --git a/environments/infura.toml b/environments/infura.toml index d9afd385..fb9876fb 100644 --- a/environments/infura.toml +++ b/environments/infura.toml @@ -4,4 +4,4 @@ port = 5432 [client] - ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" \ No newline at end of file + ipcPath = "" diff --git a/integration_test/contract_watcher_full_transformer_test.go b/integration_test/contract_watcher_full_transformer_test.go index 03e174b9..bcb7a29a 100644 --- a/integration_test/contract_watcher_full_transformer_test.go +++ b/integration_test/contract_watcher_full_transformer_test.go @@ -50,7 +50,6 @@ var _ = Describe("contractWatcher full transformer", func() { Expect(ok).To(Equal(true)) Expect(c.StartingBlock).To(Equal(int64(6194633))) - Expect(c.LastBlock).To(Equal(int64(6194634))) Expect(c.Abi).To(Equal(constants.TusdAbiString)) Expect(c.Name).To(Equal("TrueUSD")) Expect(c.Address).To(Equal(tusdAddr)) diff --git a/integration_test/contract_watcher_light_transformer_test.go b/integration_test/contract_watcher_light_transformer_test.go index 45323c10..cf9efa77 100644 --- a/integration_test/contract_watcher_light_transformer_test.go +++ b/integration_test/contract_watcher_light_transformer_test.go @@ -48,7 +48,6 @@ var _ = Describe("contractWatcher light transformer", func() { Expect(ok).To(Equal(true)) Expect(c.StartingBlock).To(Equal(int64(6194632))) - Expect(c.LastBlock).To(Equal(int64(-1))) Expect(c.Abi).To(Equal(constants.TusdAbiString)) Expect(c.Name).To(Equal("TrueUSD")) Expect(c.Address).To(Equal(tusdAddr)) @@ -203,6 +202,7 @@ var _ = Describe("contractWatcher light transformer", func() { Expect(err).ToNot(HaveOccurred()) err = t.Execute() Expect(err).ToNot(HaveOccurred()) + Expect(t.Start).To(Equal(int64(6885698))) log := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) @@ -318,7 +318,7 @@ var _ = Describe("contractWatcher light transformer", func() { Describe("Execute- against both ENS and TrueUSD", func() { BeforeEach(func() { - for i := 6885692; i < 6885702; i++ { + for i := 6885692; i <= 6885701; i++ { header, err := blockChain.GetHeaderByNumber(int64(i)) Expect(err).ToNot(HaveOccurred()) _, err = headerRepository.CreateOrUpdateHeader(header) @@ -332,6 +332,7 @@ var _ = Describe("contractWatcher light transformer", func() { Expect(err).ToNot(HaveOccurred()) err = t.Execute() Expect(err).ToNot(HaveOccurred()) + Expect(t.Start).To(Equal(int64(6885702))) newOwnerLog := test_helpers.LightNewOwnerLog{} err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) diff --git a/pkg/contract_watcher/shared/poller/poller_test.go b/integration_test/poller_test.go similarity index 92% rename from pkg/contract_watcher/shared/poller/poller_test.go rename to integration_test/poller_test.go index c4e06762..fe913176 100644 --- a/pkg/contract_watcher/shared/poller/poller_test.go +++ b/integration_test/poller_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package poller_test +package integration_test import ( "fmt" @@ -34,7 +34,7 @@ import ( var _ = Describe("Poller", func() { - var p poller.Poller + var contractPoller poller.Poller var con *contract.Contract var db *postgres.DB var bc core.BlockChain @@ -46,7 +46,7 @@ var _ = Describe("Poller", func() { Describe("Full sync mode", func() { BeforeEach(func() { db, bc = test_helpers.SetupDBandBC() - p = poller.NewPoller(bc, db, types.FullSync) + contractPoller = poller.NewPoller(bc, db, types.FullSync) }) Describe("PollContract", func() { @@ -54,10 +54,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, []string{"balanceOf"}) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -89,7 +88,7 @@ var _ = Describe("Poller", func() { Expect(len(con.Methods)).To(Equal(1)) con.AddEmittedHash(common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"), common.HexToHash("0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86")) - err := p.PollContractAt(*con, 6885877) + err := contractPoller.PollContractAt(*con, 6885877) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.Owner{} @@ -109,10 +108,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, nil) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -125,7 +123,7 @@ var _ = Describe("Poller", func() { Describe("FetchContractData", func() { It("Calls a single contract method", func() { var name = new(string) - err := p.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) + err := contractPoller.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) Expect(err).ToNot(HaveOccurred()) Expect(*name).To(Equal("TrueUSD")) }) @@ -135,7 +133,7 @@ var _ = Describe("Poller", func() { Describe("Light sync mode", func() { BeforeEach(func() { db, bc = test_helpers.SetupDBandBC() - p = poller.NewPoller(bc, db, types.LightSync) + contractPoller = poller.NewPoller(bc, db, types.LightSync) }) Describe("PollContract", func() { @@ -143,10 +141,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, []string{"balanceOf"}) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -178,7 +175,7 @@ var _ = Describe("Poller", func() { Expect(len(con.Methods)).To(Equal(1)) con.AddEmittedHash(common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"), common.HexToHash("0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86")) - err := p.PollContractAt(*con, 6885877) + err := contractPoller.PollContractAt(*con, 6885877) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.Owner{} @@ -198,10 +195,9 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupTusdContract(nil, nil) Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 - con.LastBlock = 6707323 con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6707323) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -214,11 +210,10 @@ var _ = Describe("Poller", func() { con = test_helpers.SetupENSContract(nil, []string{"resolver"}) Expect(con.Abi).To(Equal(constants.ENSAbiString)) con.StartingBlock = 6921967 - con.LastBlock = 6921968 con.EmittedAddrs = map[interface{}]bool{} con.Piping = false con.AddEmittedHash(common.HexToHash("0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8")) - err := p.PollContract(*con) + err := contractPoller.PollContract(*con, 6921968) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.Resolver{} @@ -230,10 +225,10 @@ var _ = Describe("Poller", func() { test_helpers.TearDown(db) db, bc = test_helpers.SetupDBandBC() - p = poller.NewPoller(bc, db, types.LightSync) + contractPoller = poller.NewPoller(bc, db, types.LightSync) con.Piping = true - err = p.PollContract(*con) + err = contractPoller.PollContract(*con, 6921968) Expect(err).ToNot(HaveOccurred()) err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct) @@ -248,7 +243,7 @@ var _ = Describe("Poller", func() { Describe("FetchContractData", func() { It("Calls a single contract method", func() { var name = new(string) - err := p.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) + err := contractPoller.FetchContractData(constants.TusdAbiString, constants.TusdContractAddress, "name", nil, &name, 6197514) Expect(err).ToNot(HaveOccurred()) Expect(*name).To(Equal("TrueUSD")) }) diff --git a/libraries/shared/factories/event/README.md b/libraries/shared/factories/event/README.md index 0a1aa33c..9ad2093a 100644 --- a/libraries/shared/factories/event/README.md +++ b/libraries/shared/factories/event/README.md @@ -264,7 +264,7 @@ func (repository *ExampleRepository) SetDB(db *postgres.DB) { } func (repository ExampleRepository) Create(headerID int64, models []interface{}) error { - tx, dBaseErr := repository.db.Begin() + tx, dBaseErr := repository.db.Beginx() if dBaseErr != nil { return dBaseErr } diff --git a/libraries/shared/watcher/contract_watcher.go b/libraries/shared/watcher/contract_watcher.go index 880dfb5b..ebb4cc37 100644 --- a/libraries/shared/watcher/contract_watcher.go +++ b/libraries/shared/watcher/contract_watcher.go @@ -49,7 +49,6 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error { return fmt.Errorf("initializers of type %T, not %T", inits, []transformer.ContractTransformerInitializer{}) } - watcher.Transformers = make([]transformer.ContractTransformer, 0, len(initializers)) for _, initializer := range initializers { t := initializer(watcher.DB, watcher.BlockChain) watcher.Transformers = append(watcher.Transformers, t) @@ -65,7 +64,7 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error { return nil } -func (watcher *ContractWatcher) Execute(interface{}) error { +func (watcher *ContractWatcher) Execute() error { for _, transformer := range watcher.Transformers { err := transformer.Execute() if err != nil { diff --git a/pkg/config/contract.go b/pkg/config/contract.go index a2a09d2f..50b8366a 100644 --- a/pkg/config/contract.go +++ b/pkg/config/contract.go @@ -19,6 +19,7 @@ package config import ( log "github.com/sirupsen/logrus" "github.com/spf13/viper" + "github.com/vulcanize/vulcanizedb/pkg/geth" "strings" ) @@ -65,20 +66,20 @@ type ContractConfig struct { Piping map[string]bool } -func (oc *ContractConfig) PrepConfig() { +func (contractConfig *ContractConfig) PrepConfig() { addrs := viper.GetStringSlice("contract.addresses") - oc.Network = viper.GetString("contract.network") - oc.Addresses = make(map[string]bool, len(addrs)) - oc.Abis = make(map[string]string, len(addrs)) - oc.Methods = make(map[string][]string, len(addrs)) - oc.Events = make(map[string][]string, len(addrs)) - oc.MethodArgs = make(map[string][]string, len(addrs)) - oc.EventArgs = make(map[string][]string, len(addrs)) - oc.StartingBlocks = make(map[string]int64, len(addrs)) - oc.Piping = make(map[string]bool, len(addrs)) + contractConfig.Network = viper.GetString("contract.network") + contractConfig.Addresses = make(map[string]bool, len(addrs)) + contractConfig.Abis = make(map[string]string, len(addrs)) + contractConfig.Methods = make(map[string][]string, len(addrs)) + contractConfig.Events = make(map[string][]string, len(addrs)) + contractConfig.MethodArgs = make(map[string][]string, len(addrs)) + contractConfig.EventArgs = make(map[string][]string, len(addrs)) + contractConfig.StartingBlocks = make(map[string]int64, len(addrs)) + contractConfig.Piping = make(map[string]bool, len(addrs)) // De-dupe addresses for _, addr := range addrs { - oc.Addresses[strings.ToLower(addr)] = true + contractConfig.Addresses[strings.ToLower(addr)] = true } // Iterate over addresses to pull out config info for each contract @@ -87,26 +88,29 @@ func (oc *ContractConfig) PrepConfig() { // Get and check abi var abi string - _, abiOK := transformer["abi"] + abiInterface, abiOK := transformer["abi"] if !abiOK { log.Warnf("contract %s not configured with an ABI, will attempt to fetch it from Etherscan\r\n", addr) } else { - abiInterface := transformer["abi"] abi, abiOK = abiInterface.(string) if !abiOK { log.Fatal(addr, "transformer `abi` not of type []string") } } - oc.Abis[strings.ToLower(addr)] = abi + if abi != "" { + if _, abiErr := geth.ParseAbi(abi); abiErr != nil { + log.Fatal(addr, "transformer `abi` not valid JSON") + } + } + contractConfig.Abis[strings.ToLower(addr)] = abi // Get and check events events := make([]string, 0) - _, eventsOK := transformer["events"] + eventsInterface, eventsOK := transformer["events"] if !eventsOK { log.Warnf("contract %s not configured with a list of events to watch, will watch all events\r\n", addr) events = []string{} } else { - eventsInterface := transformer["events"] eventsI, eventsOK := eventsInterface.([]interface{}) if !eventsOK { log.Fatal(addr, "transformer `events` not of type []string\r\n") @@ -119,16 +123,15 @@ func (oc *ContractConfig) PrepConfig() { events = append(events, str) } } - oc.Events[strings.ToLower(addr)] = events + contractConfig.Events[strings.ToLower(addr)] = events // Get and check methods methods := make([]string, 0) - _, methodsOK := transformer["methods"] + methodsInterface, methodsOK := transformer["methods"] if !methodsOK { log.Warnf("contract %s not configured with a list of methods to poll, will not poll any methods\r\n", addr) methods = []string{} } else { - methodsInterface := transformer["methods"] methodsI, methodsOK := methodsInterface.([]interface{}) if !methodsOK { log.Fatal(addr, "transformer `methods` not of type []string\r\n") @@ -141,16 +144,15 @@ func (oc *ContractConfig) PrepConfig() { methods = append(methods, str) } } - oc.Methods[strings.ToLower(addr)] = methods + contractConfig.Methods[strings.ToLower(addr)] = methods // Get and check eventArgs eventArgs := make([]string, 0) - _, eventArgsOK := transformer["eventArgs"] + eventArgsInterface, eventArgsOK := transformer["eventArgs"] if !eventArgsOK { log.Warnf("contract %s not configured with a list of event arguments to filter for, will not filter events for specific emitted values\r\n", addr) eventArgs = []string{} } else { - eventArgsInterface := transformer["eventArgs"] eventArgsI, eventArgsOK := eventArgsInterface.([]interface{}) if !eventArgsOK { log.Fatal(addr, "transformer `eventArgs` not of type []string\r\n") @@ -163,16 +165,15 @@ func (oc *ContractConfig) PrepConfig() { eventArgs = append(eventArgs, str) } } - oc.EventArgs[strings.ToLower(addr)] = eventArgs + contractConfig.EventArgs[strings.ToLower(addr)] = eventArgs // Get and check methodArgs methodArgs := make([]string, 0) - _, methodArgsOK := transformer["methodArgs"] + methodArgsInterface, methodArgsOK := transformer["methodArgs"] if !methodArgsOK { log.Warnf("contract %s not configured with a list of method argument values to poll with, will poll methods with all available arguments\r\n", addr) methodArgs = []string{} } else { - methodArgsInterface := transformer["methodArgs"] methodArgsI, methodArgsOK := methodArgsInterface.([]interface{}) if !methodArgsOK { log.Fatal(addr, "transformer `methodArgs` not of type []string\r\n") @@ -185,7 +186,7 @@ func (oc *ContractConfig) PrepConfig() { methodArgs = append(methodArgs, str) } } - oc.MethodArgs[strings.ToLower(addr)] = methodArgs + contractConfig.MethodArgs[strings.ToLower(addr)] = methodArgs // Get and check startingBlock startInterface, startOK := transformer["startingblock"] @@ -196,7 +197,7 @@ func (oc *ContractConfig) PrepConfig() { if !startOK { log.Fatal(addr, "transformer `startingBlock` not of type int\r\n") } - oc.StartingBlocks[strings.ToLower(addr)] = start + contractConfig.StartingBlocks[strings.ToLower(addr)] = start // Get pipping var piping bool @@ -211,6 +212,6 @@ func (oc *ContractConfig) PrepConfig() { log.Fatal(addr, "transformer `piping` not of type bool\r\n") } } - oc.Piping[strings.ToLower(addr)] = piping + contractConfig.Piping[strings.ToLower(addr)] = piping } } diff --git a/pkg/contract_watcher/full/converter/converter.go b/pkg/contract_watcher/full/converter/converter.go index 39341145..09bae256 100644 --- a/pkg/contract_watcher/full/converter/converter.go +++ b/pkg/contract_watcher/full/converter/converter.go @@ -49,10 +49,10 @@ func (c *Converter) Update(info *contract.Contract) { // Convert the given watched event log into a types.Log for the given event func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { - contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) log := helpers.ConvertToLog(watchedEvent) - err := contract.UnpackLogIntoMap(values, event.Name, log) + err := boundContract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/contract_watcher/full/transformer/transformer.go b/pkg/contract_watcher/full/transformer/transformer.go index 132292ab..ad2bba69 100644 --- a/pkg/contract_watcher/full/transformer/transformer.go +++ b/pkg/contract_watcher/full/transformer/transformer.go @@ -36,37 +36,40 @@ import ( // Requires a fully synced vDB and a running eth node (or infura) type Transformer struct { // Database interfaces - datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() - datastore.WatchedEventRepository // Watched event log views, created by the log filters - repository.EventRepository // Holds transformed watched event log data + FilterRepository datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() + WatchedEventRepository datastore.WatchedEventRepository // Watched event log views, created by the log filters + TransformedEventRepository repository.EventRepository // Holds transformed watched event log data // Pre-processing interfaces - parser.Parser // Parses events and methods out of contract abi fetched using contract address - retriever.BlockRetriever // Retrieves first block for contract and current block height + Parser parser.Parser // Parses events and methods out of contract abi fetched using contract address + Retriever retriever.BlockRetriever // Retrieves first block for contract and current block height // Processing interfaces - converter.ConverterInterface // Converts watched event logs into custom log - poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore + Converter converter.ConverterInterface // Converts watched event logs into custom log + Poller poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore // Store contract configuration information Config config.ContractConfig // Store contract info as mapping to contract address Contracts map[string]*contract.Contract + + // Latest block in the block repository + LastBlock int64 } // Transformer takes in config for blockchain, database, and network id func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer { return &Transformer{ - Poller: poller.NewPoller(BC, DB, types.FullSync), - Parser: parser.NewParser(con.Network), - BlockRetriever: retriever.NewBlockRetriever(DB), - ConverterInterface: &converter.Converter{}, - Contracts: map[string]*contract.Contract{}, - WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, - FilterRepository: repositories.FilterRepository{DB: DB}, - EventRepository: repository.NewEventRepository(DB, types.FullSync), - Config: con, + Poller: poller.NewPoller(BC, DB, types.FullSync), + Parser: parser.NewParser(con.Network), + Retriever: retriever.NewBlockRetriever(DB), + Converter: &converter.Converter{}, + Contracts: map[string]*contract.Contract{}, + WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, + FilterRepository: repositories.FilterRepository{DB: DB}, + TransformedEventRepository: repository.NewEventRepository(DB, types.FullSync), + Config: con, } } @@ -92,15 +95,10 @@ func (tr *Transformer) Init() error { } // Get first block and most recent block number in the header repo - firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock(contractAddr) + firstBlock, err := tr.Retriever.RetrieveFirstBlock(contractAddr) if err != nil { return err } - lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock() - if err != nil { - return err - } - // Set to specified range if it falls within the bounds if firstBlock < tr.Config.StartingBlocks[contractAddr] { firstBlock = tr.Config.StartingBlocks[contractAddr] @@ -108,7 +106,7 @@ func (tr *Transformer) Init() error { // Get contract name if it has one var name = new(string) - tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock) + tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock) // Remove any potential accidental duplicate inputs in arg filter values eventArgs := map[string]bool{} @@ -128,7 +126,6 @@ func (tr *Transformer) Init() error { Abi: tr.Parser.Abi(), ParsedAbi: tr.Parser.ParsedAbi(), StartingBlock: firstBlock, - LastBlock: lastBlock, Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), FilterArgs: eventArgs, @@ -144,7 +141,7 @@ func (tr *Transformer) Init() error { // Iterate over filters and push them to the repo using filter repository interface for _, filter := range info.Filters { - err = tr.CreateFilter(filter) + err = tr.FilterRepository.CreateFilter(filter) if err != nil { return err } @@ -154,6 +151,13 @@ func (tr *Transformer) Init() error { tr.Contracts[contractAddr] = info } + // Get the most recent block number in the block repo + var err error + tr.LastBlock, err = tr.Retriever.RetrieveMostRecentBlock() + if err != nil { + return err + } + return nil } @@ -169,11 +173,11 @@ func (tr *Transformer) Execute() error { // Iterate through all internal contracts for _, con := range tr.Contracts { // Update converter with current contract - tr.Update(con) + tr.Converter.Update(con) // Iterate through contract filters and get watched event logs for eventSig, filter := range con.Filters { - watchedEvents, err := tr.GetWatchedEvents(filter.Name) + watchedEvents, err := tr.WatchedEventRepository.GetWatchedEvents(filter.Name) if err != nil { return err } @@ -181,7 +185,7 @@ func (tr *Transformer) Execute() error { // Iterate over watched event logs for _, we := range watchedEvents { // Convert them to our custom log type - cstm, err := tr.ConverterInterface.Convert(*we, con.Events[eventSig]) + cstm, err := tr.Converter.Convert(*we, con.Events[eventSig]) if err != nil { return err } @@ -191,7 +195,7 @@ func (tr *Transformer) Execute() error { // If log is not empty, immediately persist in repo // Run this in seperate goroutine? - err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) + err = tr.TransformedEventRepository.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) if err != nil { return err } @@ -201,12 +205,19 @@ func (tr *Transformer) Execute() error { // After persisting all watched event logs // poller polls select contract methods // and persists the results into custom pg tables - // Run this in seperate goroutine? - if err := tr.PollContract(*con); err != nil { + if err := tr.Poller.PollContract(*con, tr.LastBlock); err != nil { return err } } + // At the end of a transformation cycle, and before the next + // update the latest block from the block repo + var err error + tr.LastBlock, err = tr.Retriever.RetrieveMostRecentBlock() + if err != nil { + return err + } + return nil } diff --git a/pkg/contract_watcher/full/transformer/transformer_test.go b/pkg/contract_watcher/full/transformer/transformer_test.go index 0cb0fb31..ac03e1a6 100644 --- a/pkg/contract_watcher/full/transformer/transformer_test.go +++ b/pkg/contract_watcher/full/transformer/transformer_test.go @@ -17,7 +17,6 @@ package transformer_test import ( - "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "math/rand" "time" @@ -27,6 +26,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types" @@ -67,7 +67,7 @@ var _ = Describe("Transformer", func() { Expect(ok).To(Equal(true)) Expect(c.StartingBlock).To(Equal(firstBlock)) - Expect(c.LastBlock).To(Equal(mostRecentBlock)) + Expect(t.LastBlock).To(Equal(mostRecentBlock)) Expect(c.Abi).To(Equal(fakeAbi)) Expect(c.Name).To(Equal(fakeContractName)) Expect(c.Address).To(Equal(fakeAddress)) @@ -90,7 +90,7 @@ func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser return transformer.Transformer{ FilterRepository: &fakes.MockFilterRepository{}, Parser: parsr, - BlockRetriever: blockRetriever, + Retriever: blockRetriever, Poller: pollr, Contracts: map[string]*contract.Contract{}, Config: mocks.MockConfig, diff --git a/pkg/contract_watcher/light/converter/converter.go b/pkg/contract_watcher/light/converter/converter.go index a1b7e051..3a62c807 100644 --- a/pkg/contract_watcher/light/converter/converter.go +++ b/pkg/contract_watcher/light/converter/converter.go @@ -48,7 +48,7 @@ func (c *Converter) Update(info *contract.Contract) { // Convert the given watched event log into a types.Log for the given event func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { - contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) returnLogs := make([]types.Log, 0, len(logs)) for _, log := range logs { values := make(map[string]interface{}) @@ -57,7 +57,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in values[field.Name] = i } - err := contract.UnpackLogIntoMap(values, event.Name, log) + err := boundContract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/contract_watcher/light/repository/header_repository.go b/pkg/contract_watcher/light/repository/header_repository.go index 47437f13..3af7652c 100644 --- a/pkg/contract_watcher/light/repository/header_repository.go +++ b/pkg/contract_watcher/light/repository/header_repository.go @@ -17,8 +17,8 @@ package repository import ( - "database/sql" "fmt" + "github.com/jmoiron/sqlx" "github.com/hashicorp/golang-lru" @@ -130,7 +130,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) // Marks all of the provided headers checked for each of the provided column ids func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { - tx, err := r.db.Begin() + tx, err := r.db.Beginx() if err != nil { return err } @@ -280,7 +280,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) { } // Used to mark a header checked as part of some external transaction so as to group into one commit -func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error { +func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) VALUES ($1, $2) ON CONFLICT (header_id) DO diff --git a/pkg/contract_watcher/light/repository/header_repository_test.go b/pkg/contract_watcher/light/repository/header_repository_test.go index 02a94bbf..01eb876a 100644 --- a/pkg/contract_watcher/light/repository/header_repository_test.go +++ b/pkg/contract_watcher/light/repository/header_repository_test.go @@ -32,7 +32,6 @@ import ( var _ = Describe("Repository", func() { var db *postgres.DB - var bc core.BlockChain var contractHeaderRepo repository.HeaderRepository // contract_watcher light header repository var coreHeaderRepo repositories.HeaderRepository // pkg/datastore header repository var eventIDs = []string{ @@ -47,7 +46,7 @@ var _ = Describe("Repository", func() { } BeforeEach(func() { - db, bc = test_helpers.SetupDBandBC() + db, _ = test_helpers.SetupDBandBC() contractHeaderRepo = repository.NewHeaderRepository(db) coreHeaderRepo = repositories.NewHeaderRepository(db) }) @@ -205,7 +204,7 @@ var _ = Describe("Repository", func() { }) It("Returns at most 100 headers", func() { - add102Headers(coreHeaderRepo, bc) + add102Headers(coreHeaderRepo) err := contractHeaderRepo.AddCheckColumns(eventIDs) Expect(err).ToNot(HaveOccurred()) @@ -359,11 +358,11 @@ func addDiscontinuousHeaders(coreHeaderRepo repositories.HeaderRepository) { coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4) } -func add102Headers(coreHeaderRepo repositories.HeaderRepository, blockChain core.BlockChain) { +func add102Headers(coreHeaderRepo repositories.HeaderRepository) { + baseHeader := mocks.MockHeader1 for i := 6194632; i < 6194733; i++ { - header, err := blockChain.GetHeaderByNumber(int64(i)) - Expect(err).ToNot(HaveOccurred()) - _, err = coreHeaderRepo.CreateOrUpdateHeader(header) + _, err := coreHeaderRepo.CreateOrUpdateHeader(baseHeader) Expect(err).ToNot(HaveOccurred()) + baseHeader.BlockNumber++ } } diff --git a/pkg/contract_watcher/light/transformer/transformer.go b/pkg/contract_watcher/light/transformer/transformer.go index 7753e286..8ce5f064 100644 --- a/pkg/contract_watcher/light/transformer/transformer.go +++ b/pkg/contract_watcher/light/transformer/transformer.go @@ -18,12 +18,12 @@ package transformer import ( "errors" - "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" gethTypes "github.com/ethereum/go-ethereum/core/types" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository" @@ -40,17 +40,17 @@ import ( // Requires a light synced vDB (headers) and a running eth node (or infura) type Transformer struct { // Database interfaces - srep.EventRepository // Holds transformed watched event log data - repository.HeaderRepository // Interface for interaction with header repositories + EventRepository srep.EventRepository // Holds transformed watched event log data + HeaderRepository repository.HeaderRepository // Interface for interaction with header repositories // Pre-processing interfaces - parser.Parser // Parses events and methods out of contract abi fetched using contract address - retriever.BlockRetriever // Retrieves first block for contract and current block height + Parser parser.Parser // Parses events and methods out of contract abi fetched using contract address + Retriever retriever.BlockRetriever // Retrieves first block for contract // Processing interfaces - fetcher.Fetcher // Fetches event logs, using header hashes - converter.ConverterInterface // Converts watched event logs into custom log - poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore + Fetcher fetcher.Fetcher // Fetches event logs, using header hashes + Converter converter.ConverterInterface // Converts watched event logs into custom log + Poller poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore // Store contract configuration information Config config.ContractConfig @@ -64,7 +64,7 @@ type Transformer struct { sortedMethodIds map[string][]string // Map to sort method column ids by contract, for post fetch method polling eventIds []string // Holds event column ids across all contract, for batch fetching of headers eventFilters []common.Hash // Holds topic0 hashes across all contracts, for batch fetching of logs - start int64 // Hold the lowest starting block and the highest ending block + Start int64 // Hold the lowest starting block and the highest ending block } // Order-of-operations: @@ -77,15 +77,15 @@ type Transformer struct { func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer { return &Transformer{ - Poller: poller.NewPoller(bc, db, types.LightSync), - Fetcher: fetcher.NewFetcher(bc), - Parser: parser.NewParser(con.Network), - HeaderRepository: repository.NewHeaderRepository(db), - BlockRetriever: retriever.NewBlockRetriever(db), - ConverterInterface: &converter.Converter{}, - Contracts: map[string]*contract.Contract{}, - EventRepository: srep.NewEventRepository(db, types.LightSync), - Config: con, + Poller: poller.NewPoller(bc, db, types.LightSync), + Fetcher: fetcher.NewFetcher(bc), + Parser: parser.NewParser(con.Network), + HeaderRepository: repository.NewHeaderRepository(db), + Retriever: retriever.NewBlockRetriever(db), + Converter: &converter.Converter{}, + Contracts: map[string]*contract.Contract{}, + EventRepository: srep.NewEventRepository(db, types.LightSync), + Config: con, } } @@ -100,7 +100,7 @@ func (tr *Transformer) Init() error { tr.sortedMethodIds = make(map[string][]string) // Map to sort method column ids by contract, for post fetch method polling tr.eventIds = make([]string, 0) // Holds event column ids across all contract, for batch fetching of headers tr.eventFilters = make([]common.Hash, 0) // Holds topic0 hashes across all contracts, for batch fetching of logs - tr.start = 100000000000 // Hold the lowest starting block and the highest ending block + tr.Start = 100000000000 // Iterate through all internal contract addresses for contractAddr := range tr.Config.Addresses { @@ -120,7 +120,7 @@ func (tr *Transformer) Init() error { } // Get first block and most recent block number in the header repo - firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock() + firstBlock, err := tr.Retriever.RetrieveFirstBlock() if err != nil { return err } @@ -132,7 +132,7 @@ func (tr *Transformer) Init() error { // Get contract name if it has one var name = new(string) - tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, -1) + tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1) // Remove any potential accidental duplicate inputs eventArgs := map[string]bool{} @@ -152,7 +152,6 @@ func (tr *Transformer) Init() error { Abi: tr.Parser.Abi(), ParsedAbi: tr.Parser.ParsedAbi(), StartingBlock: firstBlock, - LastBlock: -1, Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), FilterArgs: eventArgs, @@ -189,8 +188,8 @@ func (tr *Transformer) Init() error { } // Update start to the lowest block - if con.StartingBlock < tr.start { - tr.start = con.StartingBlock + if con.StartingBlock < tr.Start { + tr.Start = con.StartingBlock } } @@ -202,20 +201,20 @@ func (tr *Transformer) Execute() error { return errors.New("error: transformer has no initialized contracts") } - // Map to sort batch fetched logs by which contract they belong to, for post fetch processing - sortedLogs := make(map[string][]gethTypes.Log) - for _, con := range tr.Contracts { - sortedLogs[con.Address] = []gethTypes.Log{} - } - // Find unchecked headers for all events across all contracts; these are returned in asc order - missingHeaders, err := tr.HeaderRepository.MissingHeadersForAll(tr.start, -1, tr.eventIds) + missingHeaders, err := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds) if err != nil { return err } // Iterate over headers for _, header := range missingHeaders { + // Set `start` to this header + // This way if we throw an error but don't bring the execution cycle down (how it is currently handled) + // we restart the cycle at this header + tr.Start = header.BlockNumber + // Map to sort batch fetched logs by which contract they belong to, for post fetch processing + sortedLogs := make(map[string][]gethTypes.Log) // And fetch all event logs across contracts at this header allLogs, err := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header) if err != nil { @@ -233,6 +232,7 @@ func (tr *Transformer) Execute() error { if err != nil { return err } + tr.Start = header.BlockNumber + 1 // Empty header; setup to start at the next header continue } @@ -249,10 +249,10 @@ func (tr *Transformer) Execute() error { } // Configure converter with this contract con := tr.Contracts[conAddr] - tr.ConverterInterface.Update(con) + tr.Converter.Update(con) // Convert logs into batches of log mappings (eventName => []types.Logs - convertedLogs, err := tr.ConverterInterface.ConvertBatch(logs, con.Events, header.Id) + convertedLogs, err := tr.Converter.ConvertBatch(logs, con.Events, header.Id) if err != nil { return err } @@ -281,8 +281,8 @@ func (tr *Transformer) Execute() error { if err != nil { return err } - - tr.start = header.BlockNumber + 1 + // Success; setup to start at the next header + tr.Start = header.BlockNumber + 1 } return nil diff --git a/pkg/contract_watcher/light/transformer/transformer_test.go b/pkg/contract_watcher/light/transformer/transformer_test.go index 98a31722..8c17f570 100644 --- a/pkg/contract_watcher/light/transformer/transformer_test.go +++ b/pkg/contract_watcher/light/transformer/transformer_test.go @@ -19,6 +19,7 @@ package transformer_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer" "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract" @@ -54,7 +55,47 @@ var _ = Describe("Transformer", func() { Expect(ok).To(Equal(true)) Expect(c.StartingBlock).To(Equal(firstBlock)) - Expect(c.LastBlock).To(Equal(int64(-1))) + Expect(c.Abi).To(Equal(fakeAbi)) + Expect(c.Name).To(Equal(fakeContractName)) + Expect(c.Address).To(Equal(fakeAddress)) + }) + + It("Fails to initialize if first block cannot be fetched from vDB headers table", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + blockRetriever.FirstBlockErr = fakes.FakeError + t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) + + err := t.Init() + + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(fakes.FakeError)) + }) + }) + + Describe("Execute", func() { + It("Executes contract transformations", func() { + blockRetriever := &fakes.MockLightBlockRetriever{} + firstBlock := int64(1) + blockRetriever.FirstBlock = firstBlock + + parsr := &fakes.MockParser{} + fakeAbi := "fake_abi" + parsr.AbiToReturn = fakeAbi + + pollr := &fakes.MockPoller{} + fakeContractName := "fake_contract_name" + pollr.ContractName = fakeContractName + + t := getFakeTransformer(blockRetriever, parsr, pollr) + + err := t.Init() + + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[fakeAddress] + Expect(ok).To(Equal(true)) + + Expect(c.StartingBlock).To(Equal(firstBlock)) Expect(c.Abi).To(Equal(fakeAbi)) Expect(c.Name).To(Equal(fakeContractName)) Expect(c.Address).To(Equal(fakeAddress)) @@ -76,7 +117,7 @@ var _ = Describe("Transformer", func() { func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { return transformer.Transformer{ Parser: parsr, - BlockRetriever: blockRetriever, + Retriever: blockRetriever, Poller: pollr, HeaderRepository: &fakes.MockLightHeaderRepository{}, Contracts: map[string]*contract.Contract{}, diff --git a/pkg/contract_watcher/shared/contract/contract.go b/pkg/contract_watcher/shared/contract/contract.go index b3eb3b7f..6b3eb0ff 100644 --- a/pkg/contract_watcher/shared/contract/contract.go +++ b/pkg/contract_watcher/shared/contract/contract.go @@ -34,7 +34,6 @@ type Contract struct { Address string // Address of the contract Network string // Network on which the contract is deployed; default empty "" is Ethereum mainnet StartingBlock int64 // Starting block of the contract - LastBlock int64 // Most recent block on the network Abi string // Abi string ParsedAbi abi.ABI // Parsed abi Events map[string]types.Event // List of events to watch diff --git a/pkg/contract_watcher/shared/helpers/test_helpers/database.go b/pkg/contract_watcher/shared/helpers/test_helpers/database.go index 7c297bef..93c79459 100644 --- a/pkg/contract_watcher/shared/helpers/test_helpers/database.go +++ b/pkg/contract_watcher/shared/helpers/test_helpers/database.go @@ -168,7 +168,6 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract Abi: p.Abi(), ParsedAbi: p.ParsedAbi(), StartingBlock: 6194634, - LastBlock: 6507323, Events: p.GetEvents(wantedEvents), Methods: p.GetSelectMethods(wantedMethods), MethodArgs: map[string]bool{}, @@ -215,7 +214,6 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract { Abi: p.Abi(), ParsedAbi: p.ParsedAbi(), StartingBlock: 6194634, - LastBlock: 6507323, Events: p.GetEvents(wantedEvents), Methods: p.GetSelectMethods(wantedMethods), MethodArgs: map[string]bool{}, @@ -224,7 +222,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract { } func TearDown(db *postgres.DB) { - tx, err := db.Begin() + tx, err := db.Beginx() Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM blocks`) diff --git a/pkg/contract_watcher/shared/parser/parser.go b/pkg/contract_watcher/shared/parser/parser.go index cc3da5bd..77af237b 100644 --- a/pkg/contract_watcher/shared/parser/parser.go +++ b/pkg/contract_watcher/shared/parser/parser.go @@ -98,7 +98,7 @@ func (p *parser) lookUp(contractAddr string) (string, error) { return v, nil } - return "", errors.New("ABI not present in lookup tabe") + return "", errors.New("ABI not present in lookup table") } // Returns only specified methods, if they meet the criteria diff --git a/pkg/contract_watcher/shared/poller/poller.go b/pkg/contract_watcher/shared/poller/poller.go index ae4f1974..b3f1c51b 100644 --- a/pkg/contract_watcher/shared/poller/poller.go +++ b/pkg/contract_watcher/shared/poller/poller.go @@ -34,7 +34,7 @@ import ( ) type Poller interface { - PollContract(con contract.Contract) error + PollContract(con contract.Contract, lastBlock int64) error PollContractAt(con contract.Contract, blockNumber int64) error FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error } @@ -52,8 +52,8 @@ func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *po } } -func (p *poller) PollContract(con contract.Contract) error { - for i := con.StartingBlock; i <= con.LastBlock; i++ { +func (p *poller) PollContract(con contract.Contract, lastBlock int64) error { + for i := con.StartingBlock; i <= lastBlock; i++ { if err := p.PollContractAt(con, i); err != nil { return err } diff --git a/pkg/contract_watcher/shared/poller/poller_suite_test.go b/pkg/contract_watcher/shared/poller/poller_suite_test.go deleted file mode 100644 index 9b4e94b5..00000000 --- a/pkg/contract_watcher/shared/poller/poller_suite_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package poller_test - -import ( - "github.com/sirupsen/logrus" - "io/ioutil" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestPoller(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Poller Suite Test") -} - -var _ = BeforeSuite(func() { - logrus.SetOutput(ioutil.Discard) -}) diff --git a/pkg/contract_watcher/shared/repository/event_repository.go b/pkg/contract_watcher/shared/repository/event_repository.go index 742ca9ad..14959a49 100644 --- a/pkg/contract_watcher/shared/repository/event_repository.go +++ b/pkg/contract_watcher/shared/repository/event_repository.go @@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c // Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB) func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - tx, err := r.db.Begin() + tx, err := r.db.Beginx() if err != nil { return err } @@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB) func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - tx, err := r.db.Begin() + tx, err := r.db.Beginx() if err != nil { return err } diff --git a/pkg/contract_watcher/shared/repository/method_repository.go b/pkg/contract_watcher/shared/repository/method_repository.go index 4747f141..2616675e 100644 --- a/pkg/contract_watcher/shared/repository/method_repository.go +++ b/pkg/contract_watcher/shared/repository/method_repository.go @@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ // Creates a custom postgres command to persist logs for the given event func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { - tx, err := r.DB.Begin() + tx, err := r.DB.Beginx() if err != nil { return err } diff --git a/pkg/fakes/mock_poller.go b/pkg/fakes/mock_poller.go index ab7287ee..2d782b18 100644 --- a/pkg/fakes/mock_poller.go +++ b/pkg/fakes/mock_poller.go @@ -8,7 +8,7 @@ type MockPoller struct { ContractName string } -func (*MockPoller) PollContract(con contract.Contract) error { +func (*MockPoller) PollContract(con contract.Contract, lastBlock int64) error { panic("implement me") } diff --git a/pkg/plugin/test_helpers/database.go b/pkg/plugin/test_helpers/database.go index 0ab5c63f..345683f0 100644 --- a/pkg/plugin/test_helpers/database.go +++ b/pkg/plugin/test_helpers/database.go @@ -54,7 +54,7 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) { } func TearDown(db *postgres.DB) { - tx, err := db.Begin() + tx, err := db.Beginx() Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM headers`) diff --git a/test_config/test_config.go b/test_config/test_config.go index b64a9278..0f0071ae 100644 --- a/test_config/test_config.go +++ b/test_config/test_config.go @@ -17,6 +17,7 @@ package test_config import ( + "errors" "fmt" "os" @@ -48,10 +49,10 @@ func setTestConfig() { TestConfig.SetConfigName("private") TestConfig.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/") err := TestConfig.ReadInConfig() - ipc := TestConfig.GetString("client.ipcPath") if err != nil { log.Fatal(err) } + ipc := TestConfig.GetString("client.ipcPath") hn := TestConfig.GetString("database.hostname") port := TestConfig.GetInt("database.port") name := TestConfig.GetString("database.name") @@ -71,10 +72,20 @@ func setInfuraConfig() { Infura.SetConfigName("infura") Infura.AddConfigPath("$GOPATH/src/github.com/vulcanize/vulcanizedb/environments/") err := Infura.ReadInConfig() - ipc := Infura.GetString("client.ipcpath") if err != nil { log.Fatal(err) } + ipc := Infura.GetString("client.ipcpath") + + // If we don't have an ipc path in the config file, check the env variable + if ipc == "" { + Infura.BindEnv("url", "INFURA_URL") + ipc = Infura.GetString("url") + } + if ipc == "" { + log.Fatal(errors.New("infura.toml IPC path or $INFURA_URL env variable need to be set")) + } + InfuraClient = config.Client{ IPCPath: ipc, }