From 8174ce4aee6172067cf0d5eb90597fef7d15ae90 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Mon, 11 Mar 2019 11:19:18 -0500 Subject: [PATCH] refactor omni code; rename omniWatcher => contractWatcher; use config instead of excessive number of CLI flags --- cmd/contractWatcher.go | 123 ++++ cmd/omniWatcher.go | 121 ---- .../omni_full_transformer_test.go | 75 +-- .../omni_light_transformer_test.go | 105 ++-- pkg/config/contract.go | 177 ++++++ pkg/omni/full/converter/converter.go | 4 +- pkg/omni/full/transformer/transformer.go | 164 ++---- pkg/omni/full/transformer/transformer_test.go | 393 +++++++++---- pkg/omni/light/converter/converter.go | 8 +- .../light/repository/header_repository.go | 6 +- .../light/retriever/retriever_suite_test.go | 2 +- pkg/omni/light/transformer/transformer.go | 146 ++--- .../light/transformer/transformer_test.go | 526 ++++++++++++++---- pkg/omni/shared/constants/constants.go | 1 - pkg/omni/shared/contract/contract.go | 6 - .../shared/helpers/test_helpers/database.go | 12 +- .../helpers/test_helpers/mocks/entities.go | 77 +++ .../shared/repository/event_repository.go | 4 +- .../shared/repository/method_repository.go | 2 +- pkg/omni/shared/transformer/interface.go | 32 -- 20 files changed, 1278 insertions(+), 706 deletions(-) create mode 100644 cmd/contractWatcher.go delete mode 100644 cmd/omniWatcher.go create mode 100644 pkg/config/contract.go delete mode 100644 pkg/omni/shared/transformer/interface.go diff --git a/cmd/contractWatcher.go b/cmd/contractWatcher.go new file mode 100644 index 00000000..342de614 --- /dev/null +++ b/cmd/contractWatcher.go @@ -0,0 +1,123 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer" + ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" + lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" + "github.com/vulcanize/vulcanizedb/utils" +) + +// contractWatcherCmd represents the contractWatcher command +var contractWatcherCmd = &cobra.Command{ + Use: "contractWatcher", + Short: "Watches events at the provided contract address using fully synced vDB", + Long: `Uses input contract address and event filters to watch events + +Expects an ethereum node to be running +Expects an archival node synced into vulcanizeDB +Requires a .toml config file: + + [database] + name = "vulcanize_public" + hostname = "localhost" + port = 5432 + + [client] + ipcPath = "https://mainnet.infura.io/J5Vd2fRtGsw0zZ0Ov3BL" + + [contract] + network = "" + addresses = [ + "contractAddress1", + "contractAddress2" + ] + [contract.contractAddress1] + abi = 'ABI for contract 1' + startingBlock = 982463 + [contract.contractAddress2] + abi = 'ABI for contract 2' + events = [ + "event1", + "event2" + ] + eventArgs = [ + "arg1", + "arg2" + ] + methods = [ + "method1", + "method2" + ] + methodArgs = [ + "arg1", + "arg2" + ] + startingBlock = 4448566 + piping = true +`, + Run: func(cmd *cobra.Command, args []string) { + contractWatcher() + }, +} + +var ( + mode string +) + +func contractWatcher() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + blockChain := getBlockChain() + db := utils.LoadPostgres(databaseConfig, blockChain.Node()) + + var t st.GenericTransformer + con := config.ContractConfig{} + con.PrepConfig() + switch mode { + case "light": + t = lt.NewTransformer(con, blockChain, &db) + case "full": + t = ft.NewTransformer(con, blockChain, &db) + default: + log.Fatal("Invalid mode") + } + + err := t.Init() + if err != nil { + log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) + } + + for range ticker.C { + t.Execute() + } +} + +func init() { + rootCmd.AddCommand(contractWatcherCmd) + + contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") +} diff --git a/cmd/omniWatcher.go b/cmd/omniWatcher.go deleted file mode 100644 index a07108a9..00000000 --- a/cmd/omniWatcher.go +++ /dev/null @@ -1,121 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package cmd - -import ( - "fmt" - "log" - "time" - - "github.com/spf13/cobra" - - ft "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - lt "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - st "github.com/vulcanize/vulcanizedb/pkg/omni/shared/transformer" - "github.com/vulcanize/vulcanizedb/utils" -) - -// omniWatcherCmd represents the omniWatcher command -var omniWatcherCmd = &cobra.Command{ - Use: "omniWatcher", - Short: "Watches events at the provided contract address using fully synced vDB", - Long: `Uses input contract address and event filters to watch events - -Expects an ethereum node to be running -Expects an archival node synced into vulcanizeDB -Requires a .toml config file: - - [database] - name = "vulcanize_public" - hostname = "localhost" - port = 5432 - - [client] - ipcPath = "/Users/user/Library/Ethereum/geth.ipc" -`, - Run: func(cmd *cobra.Command, args []string) { - omniWatcher() - }, -} - -var ( - network string - contractAddress string - contractAddresses []string - contractEvents []string - contractMethods []string - eventArgs []string - methodArgs []string - methodPiping bool - mode string -) - -func omniWatcher() { - if contractAddress == "" && len(contractAddresses) == 0 { - log.Fatal("Contract address required") - } - - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - blockChain := getBlockChain() - db := utils.LoadPostgres(databaseConfig, blockChain.Node()) - - var t st.Transformer - switch mode { - case "light": - t = lt.NewTransformer(network, blockChain, &db) - case "full": - t = ft.NewTransformer(network, blockChain, &db) - default: - log.Fatal("Invalid mode") - } - - contractAddresses = append(contractAddresses, contractAddress) - for _, addr := range contractAddresses { - t.SetEvents(addr, contractEvents) - t.SetMethods(addr, contractMethods) - t.SetEventArgs(addr, eventArgs) - t.SetMethodArgs(addr, methodArgs) - t.SetPiping(addr, methodPiping) - t.SetStartingBlock(addr, startingBlockNumber) - } - - err := t.Init() - if err != nil { - log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) - } - - for range ticker.C { - t.Execute() - } -} - -func init() { - rootCmd.AddCommand(omniWatcherCmd) - - omniWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)") - omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") - omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address") - omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched") - omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled") - omniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified") - omniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here") - omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) - omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists") - omniWatcherCmd.Flags().BoolVarP(&methodPiping, "piping", "p", false, "Turn on method output piping: methods listed first will be polled first and their output used as input to subsequent methods") -} diff --git a/integration_test/omni_full_transformer_test.go b/integration_test/omni_full_transformer_test.go index 5346584d..cdfb93ae 100644 --- a/integration_test/omni_full_transformer_test.go +++ b/integration_test/omni_full_transformer_test.go @@ -2,6 +2,7 @@ package integration import ( "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "math/rand" "strings" "time" @@ -41,8 +42,7 @@ var _ = Describe("Omni full transformer", func() { It("Initializes transformer's contract objects", func() { blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -64,9 +64,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -85,9 +83,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -119,9 +120,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -145,9 +149,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) @@ -161,9 +163,7 @@ var _ = Describe("Omni full transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -183,9 +183,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -210,9 +213,12 @@ var _ = Describe("Omni full transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -235,10 +241,12 @@ var _ = Describe("Omni full transformer", func() { }) It("It does not perist events if they do not pass the emitted arg filter", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -252,10 +260,15 @@ var _ = Describe("Omni full transformer", func() { }) It("If a method arg filter is applied, only those arguments are used in polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, + } + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) diff --git a/integration_test/omni_light_transformer_test.go b/integration_test/omni_light_transformer_test.go index 5634ba14..6cc8bb93 100644 --- a/integration_test/omni_light_transformer_test.go +++ b/integration_test/omni_light_transformer_test.go @@ -2,6 +2,7 @@ package integration import ( "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" @@ -39,8 +40,7 @@ var _ = Describe("Omnit light transformer", func() { It("Initializes transformer's contract objects", func() { headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -70,9 +70,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -89,9 +87,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) c, ok := t.Contracts[tusdAddr] @@ -131,9 +132,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given methods using generated token holder address", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -150,9 +154,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Fails if initialization has not been done", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) err = t.Execute() Expect(err).To(HaveOccurred()) }) @@ -173,9 +175,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -192,9 +192,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) c, ok := t.Contracts[ensAddr] @@ -218,9 +221,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given method using list of collected hashes", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -242,10 +248,12 @@ var _ = Describe("Omnit light transformer", func() { }) It("It does not persist events if they do not pass the emitted arg filter", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -257,10 +265,15 @@ var _ = Describe("Omnit light transformer", func() { }) It("If a method arg filter is applied, only those arguments are used in polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}) + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, + } + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -302,11 +315,7 @@ var _ = Describe("Omnit light transformer", func() { }) It("Transforms watched contract data into custom repositories", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, nil) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() @@ -332,11 +341,13 @@ var _ = Describe("Omnit light transformer", func() { }) It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) ens, ok := t.Contracts[ensAddr] @@ -376,11 +387,13 @@ var _ = Describe("Omnit light transformer", func() { }) It("Polls given methods for each contract, using list of collected values", func() { - t := transformer.NewTransformer("", blockChain, db) - t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"}) - t.SetMethods(constants.EnsContractAddress, []string{"owner"}) - t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) err = t.Init() Expect(err).ToNot(HaveOccurred()) err = t.Execute() diff --git a/pkg/config/contract.go b/pkg/config/contract.go new file mode 100644 index 00000000..a55933ea --- /dev/null +++ b/pkg/config/contract.go @@ -0,0 +1,177 @@ +// VulcanizeDB +// Copyright © 2018 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "strings" +) + +// Config struct for generic contract transformer +type ContractConfig struct { + // Name for the transformer + Name string + + // Ethereum network name; default "" is mainnet + Network string + + // List of contract addresses (map to ensure no duplicates) + Addresses map[string]bool + + // Map of contract address to abi + // If an address has no associated abi the parser will attempt to fetch one from etherscan + Abis map[string]string + + // Map of contract address to slice of events + // Used to set which addresses to watch + // If any events are listed in the slice only those will be watched + // Otherwise all events in the contract ABI are watched + Events map[string][]string + + // Map of contract address to slice of methods + // If any methods are listed in the slice only those will be polled + // Otherwise no methods will be polled + Methods map[string][]string + + // Map of contract address to slice of event arguments to filter for + // If arguments are provided then only events which emit those arguments are watched + // Otherwise arguments are not filtered on events + EventArgs map[string][]string + + // Map of contract address to slice of method arguments to limit polling to + // If arguments are provided then only those arguments are allowed as arguments in method polling + // Otherwise any argument of the right type seen emitted from events at that contract will be used in method polling + MethodArgs map[string][]string + + // Map of contract address to their starting block + StartingBlocks map[string]int64 + + // Map of contract address to whether or not to pipe method polling results forward into subsequent method calls + Piping map[string]bool +} + +func (oc *ContractConfig) PrepConfig() { + addrs := viper.GetStringSlice("contract.addresses") + oc.Network = viper.GetString("contract.network") + oc.Addresses = make(map[string]bool, len(addrs)) + oc.Abis = make(map[string]string, len(addrs)) + oc.Methods = make(map[string][]string, len(addrs)) + oc.EventArgs = make(map[string][]string, len(addrs)) + oc.MethodArgs = make(map[string][]string, len(addrs)) + oc.EventArgs = make(map[string][]string, len(addrs)) + oc.StartingBlocks = make(map[string]int64, len(addrs)) + oc.Piping = make(map[string]bool, len(addrs)) + // De-dupe addresses + for _, addr := range addrs { + oc.Addresses[strings.ToLower(addr)] = true + } + + // Iterate over addresses to pull out config info for each contract + for _, addr := range addrs { + transformer := viper.GetStringMap("contract." + addr) + + // Get and check abi + abi, abiOK := transformer["abi"] + if !abiOK || abi == nil { + log.Fatal(addr, "transformer config is missing `abi` value") + } + abiRef, abiOK := abi.(string) + if !abiOK { + log.Fatal(addr, "transformer `events` not of type []string") + } + oc.Abis[strings.ToLower(addr)] = abiRef + + // Get and check events + events, eventsOK := transformer["events"] + if !eventsOK || events == nil { + log.Fatal(addr, "transformer config is missing `events` value") + } + eventsRef, eventsOK := events.([]string) + if !eventsOK { + log.Fatal(addr, "transformer `events` not of type []string") + } + if eventsRef == nil { + eventsRef = []string{} + } + oc.Events[strings.ToLower(addr)] = eventsRef + + // Get and check methods + methods, methodsOK := transformer["methods"] + if !methodsOK || methods == nil { + log.Fatal(addr, "transformer config is missing `methods` value") + } + methodsRef, methodsOK := methods.([]string) + if !methodsOK { + log.Fatal(addr, "transformer `methods` not of type []string") + } + if methodsRef == nil { + methodsRef = []string{} + } + oc.Methods[strings.ToLower(addr)] = methodsRef + + // Get and check eventArgs + eventArgs, eventArgsOK := transformer["eventArgs"] + if !eventArgsOK || eventArgs == nil { + log.Fatal(addr, "transformer config is missing `eventArgs` value") + } + eventArgsRef, eventArgsOK := eventArgs.([]string) + if !eventArgsOK { + log.Fatal(addr, "transformer `eventArgs` not of type []string") + } + if eventArgsRef == nil { + eventArgsRef = []string{} + } + oc.EventArgs[strings.ToLower(addr)] = eventArgsRef + + // Get and check methodArgs + methodArgs, methodArgsOK := transformer["methodArgs"] + if !methodArgsOK || methodArgs == nil { + log.Fatal(addr, "transformer config is missing `methodArgs` value") + } + methodArgsRef, methodArgsOK := methodArgs.([]string) + if !methodArgsOK { + log.Fatal(addr, "transformer `methodArgs` not of type []string") + } + if methodArgsRef == nil { + methodArgsRef = []string{} + } + oc.MethodArgs[strings.ToLower(addr)] = methodArgsRef + + // Get and check startingBlock + start, startOK := transformer["startingBlock"] + if !startOK || start == nil { + log.Fatal(addr, "transformer config is missing `startingBlock` value") + } + startRef, startOK := start.(int64) + if !startOK { + log.Fatal(addr, "transformer `startingBlock` not of type int") + } + oc.StartingBlocks[strings.ToLower(addr)] = startRef + + // Get pipping + pipe, pipeOK := transformer["pipping"] + if !pipeOK || pipe == nil { + log.Fatal(addr, "transformer config is missing `pipping` value") + } + pipeRef, pipeOK := pipe.(bool) + if !pipeOK { + log.Fatal(addr, "transformer `piping` not of type bool") + } + oc.Piping[strings.ToLower(addr)] = pipeRef + } +} diff --git a/pkg/omni/full/converter/converter.go b/pkg/omni/full/converter/converter.go index 2810438a..1c615c33 100644 --- a/pkg/omni/full/converter/converter.go +++ b/pkg/omni/full/converter/converter.go @@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) { // Convert the given watched event log into a types.Log for the given event func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) log := helpers.ConvertToLog(watchedEvent) - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/omni/full/transformer/transformer.go b/pkg/omni/full/transformer/transformer.go index 3a37c084..6d51bdb1 100644 --- a/pkg/omni/full/transformer/transformer.go +++ b/pkg/omni/full/transformer/transformer.go @@ -18,8 +18,8 @@ package transformer import ( "errors" - "strings" + "github.com/vulcanize/vulcanizedb/pkg/config" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -34,7 +34,7 @@ import ( ) // Requires a fully synced vDB and a running eth node (or infura) -type Transformer struct { +type transformer struct { // Database interfaces datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters() datastore.WatchedEventRepository // Watched event log views, created by the log filters @@ -48,52 +48,25 @@ type Transformer struct { converter.Converter // Converts watched event logs into custom log poller.Poller // Polls methods using contract's token holder addresses and persists them using method datastore - // Ethereum network name; default "" is mainnet - Network string + // Store contract configuration information + Config config.ContractConfig // Store contract info as mapping to contract address Contracts map[string]*contract.Contract - - // Targeted subset of events/methods - // Stored as map sof contract address to events/method names of interest - WatchedEvents map[string][]string // Default/empty event list means all are watched - WantedMethods map[string][]string // Default/empty method list means none are polled - - // Starting block for contracts - ContractStart map[string]int64 - - // Lists of addresses to filter event or method data - // before persisting; if empty no filter is applied - EventArgs map[string][]string - MethodArgs map[string][]string - - // Whether or not to create a list of emitted address or hashes for the contract in postgres - CreateAddrList map[string]bool - CreateHashList map[string]bool - - // Method piping on/off for a contract - Piping map[string]bool } // Transformer takes in config for blockchain, database, and network id -func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer { - return &Transformer{ +func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *transformer { + return &transformer{ Poller: poller.NewPoller(BC, DB, types.FullSync), - Parser: parser.NewParser(network), + Parser: parser.NewParser(con.Network), BlockRetriever: retriever.NewBlockRetriever(DB), Converter: converter.NewConverter(&contract.Contract{}), Contracts: map[string]*contract.Contract{}, WatchedEventRepository: repositories.WatchedEventRepository{DB: DB}, FilterRepository: repositories.FilterRepository{DB: DB}, EventRepository: repository.NewEventRepository(DB, types.FullSync), - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - Piping: map[string]bool{}, + Config: con, } } @@ -101,59 +74,66 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transf // Loops over all of the addr => filter sets // Uses parser to pull event info from abi // Use this info to generate event filters -func (transformer *Transformer) Init() error { - for contractAddr, subset := range transformer.WatchedEvents { - // Get Abi - err := transformer.Parser.Parse(contractAddr) - if err != nil { - return err +func (tr *transformer) Init() error { + for contractAddr := range tr.Config.Addresses { + // Configure Abi + if tr.Config.Abis[contractAddr] == "" { + // If no abi is given in the config, this method will try fetching from internal look-up table and etherscan + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + } else { + // If we have an abi from the config, load that into the parser + err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr]) + if err != nil { + return err + } } // Get first block and most recent block number in the header repo - firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr) + firstBlock, err := tr.BlockRetriever.RetrieveFirstBlock(contractAddr) if err != nil { return err } - lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock() + lastBlock, err := tr.BlockRetriever.RetrieveMostRecentBlock() if err != nil { return err } // Set to specified range if it falls within the bounds - if firstBlock < transformer.ContractStart[contractAddr] { - firstBlock = transformer.ContractStart[contractAddr] + if firstBlock < tr.Config.StartingBlocks[contractAddr] { + firstBlock = tr.Config.StartingBlocks[contractAddr] } // Get contract name if it has one var name = new(string) - transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock) + tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) // Remove any potential accidental duplicate inputs in arg filter values eventArgs := map[string]bool{} - for _, arg := range transformer.EventArgs[contractAddr] { + for _, arg := range tr.Config.EventArgs[contractAddr] { eventArgs[arg] = true } methodArgs := map[string]bool{} - for _, arg := range transformer.MethodArgs[contractAddr] { + for _, arg := range tr.Config.MethodArgs[contractAddr] { methodArgs[arg] = true } // Aggregate info into contract object info := contract.Contract{ - Name: *name, - Network: transformer.Network, - Address: contractAddr, - Abi: transformer.Parser.Abi(), - ParsedAbi: transformer.Parser.ParsedAbi(), - StartingBlock: firstBlock, - LastBlock: lastBlock, - Events: transformer.Parser.GetEvents(subset), - Methods: transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]), - FilterArgs: eventArgs, - MethodArgs: methodArgs, - CreateAddrList: transformer.CreateAddrList[contractAddr], - CreateHashList: transformer.CreateHashList[contractAddr], - Piping: transformer.Piping[contractAddr], + Name: *name, + Network: tr.Config.Network, + Address: contractAddr, + Abi: tr.Parser.Abi(), + ParsedAbi: tr.Parser.ParsedAbi(), + StartingBlock: firstBlock, + LastBlock: lastBlock, + Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), + Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + Piping: tr.Config.Piping[contractAddr], }.Init() // Use info to create filters @@ -164,14 +144,14 @@ func (transformer *Transformer) Init() error { // Iterate over filters and push them to the repo using filter repository interface for _, filter := range info.Filters { - err = transformer.FilterRepository.CreateFilter(filter) + err = tr.CreateFilter(filter) if err != nil { return err } } // Store contract info for further processing - transformer.Contracts[contractAddr] = info + tr.Contracts[contractAddr] = info } return nil @@ -182,18 +162,18 @@ func (transformer *Transformer) Init() error { // Uses converter to convert logs into custom log type // Persists converted logs into custuom postgres tables // Calls selected methods, using token holder address generated during event log conversion -func (transformer Transformer) Execute() error { - if len(transformer.Contracts) == 0 { +func (tr *transformer) Execute() error { + if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts to work with") } // Iterate through all internal contracts - for _, con := range transformer.Contracts { + for _, con := range tr.Contracts { // Update converter with current contract - transformer.Update(con) + tr.Update(con) // Iterate through contract filters and get watched event logs for eventSig, filter := range con.Filters { - watchedEvents, err := transformer.GetWatchedEvents(filter.Name) + watchedEvents, err := tr.GetWatchedEvents(filter.Name) if err != nil { return err } @@ -201,7 +181,7 @@ func (transformer Transformer) Execute() error { // Iterate over watched event logs for _, we := range watchedEvents { // Convert them to our custom log type - cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig]) + cstm, err := tr.Converter.Convert(*we, con.Events[eventSig]) if err != nil { return err } @@ -211,7 +191,7 @@ func (transformer Transformer) Execute() error { // If log is not empty, immediately persist in repo // Run this in seperate goroutine? - err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) + err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name) if err != nil { return err } @@ -222,7 +202,7 @@ func (transformer Transformer) Execute() error { // poller polls select contract methods // and persists the results into custom pg tables // Run this in seperate goroutine? - if err := transformer.PollContract(*con); err != nil { + if err := tr.PollContract(*con); err != nil { return err } } @@ -230,42 +210,6 @@ func (transformer Transformer) Execute() error { return nil } -// Used to set which contract addresses and which of their events to watch -func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) { - transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to watch events for -func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) { - transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set which contract addresses and which of their methods to call -func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) { - transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to poll methods on -func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) { - transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set the block range to watch for a given address -func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) { - transformer.ContractStart[strings.ToLower(contractAddr)] = start -} - -// Used to set whether or not to persist an account address list -func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) { - transformer.CreateAddrList[strings.ToLower(contractAddr)] = on -} - -// Used to set whether or not to persist an hash list -func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) { - transformer.CreateHashList[strings.ToLower(contractAddr)] = on -} - -// Used to turn method piping on for a contract -func (transformer *Transformer) SetPiping(contractAddr string, on bool) { - transformer.Piping[strings.ToLower(contractAddr)] = on +func (tr *transformer) GetConfig() config.ContractConfig { + return tr.Config } diff --git a/pkg/omni/full/transformer/transformer_test.go b/pkg/omni/full/transformer/transformer_test.go index f25ca8f6..e302b6f5 100644 --- a/pkg/omni/full/transformer/transformer_test.go +++ b/pkg/omni/full/transformer/transformer_test.go @@ -17,160 +17,307 @@ package transformer_test import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" "math/rand" + "strings" "time" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Transformer", func() { - var fakeAddress = "0x1234567890abcdef" + var db *postgres.DB + var err error + var blockChain core.BlockChain + var blockRepository repositories.BlockRepository + var ensAddr = strings.ToLower(constants.EnsContractAddress) + var tusdAddr = strings.ToLower(constants.TusdContractAddress) rand.Seed(time.Now().UnixNano()) - Describe("SetEvents", func() { - It("Sets which events to watch from the given contract address", func() { - watchedEvents := []string{"Transfer", "Mint"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, watchedEvents) - Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents)) - }) + BeforeEach(func() { + db, blockChain = test_helpers.SetupDBandBC() + blockRepository = *repositories.NewBlockRepository(db) }) - Describe("SetEventAddrs", func() { - It("Sets which account addresses to watch events for", func() { - eventAddrs := []string{"test1", "test2"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEventArgs(fakeAddress, eventAddrs) - Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs)) - }) - }) - - Describe("SetMethods", func() { - It("Sets which methods to poll at the given contract address", func() { - watchedMethods := []string{"balanceOf", "totalSupply"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethods(fakeAddress, watchedMethods) - Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods)) - }) - }) - - Describe("SetMethodAddrs", func() { - It("Sets which account addresses to poll methods against", func() { - methodAddrs := []string{"test1", "test2"} - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethodArgs(fakeAddress, methodAddrs) - Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs)) - }) - }) - - Describe("SetStartingBlock", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetStartingBlock(fakeAddress, 11) - Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11))) - }) - }) - - Describe("SetCreateAddrList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateAddrList(fakeAddress, true) - Expect(t.CreateAddrList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("SetCreateHashList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateHashList(fakeAddress, true) - Expect(t.CreateHashList[fakeAddress]).To(Equal(true)) - }) + AfterEach(func() { + test_helpers.TearDown(db) }) Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRetriever := &fakes.MockFullBlockRetriever{} - firstBlock := int64(1) - mostRecentBlock := int64(2) - blockRetriever.FirstBlock = firstBlock - blockRetriever.MostRecentBlock = mostRecentBlock - - parsr := &fakes.MockParser{} - fakeAbi := "fake_abi" - eventName := "Transfer" - event := types.Event{} - parsr.AbiToReturn = fakeAbi - parsr.EventName = eventName - parsr.Event = event - - pollr := &fakes.MockPoller{} - fakeContractName := "fake_contract_name" - pollr.ContractName = fakeContractName - - t := getTransformer(blockRetriever, parsr, pollr) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[fakeAddress] + c, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(true)) - Expect(c.StartingBlock).To(Equal(firstBlock)) - Expect(c.LastBlock).To(Equal(mostRecentBlock)) - Expect(c.Abi).To(Equal(fakeAbi)) - Expect(c.Name).To(Equal(fakeContractName)) - Expect(c.Address).To(Equal(fakeAddress)) + Expect(c.StartingBlock).To(Equal(int64(6194633))) + Expect(c.LastBlock).To(Equal(int64(6194634))) + Expect(c.Abi).To(Equal(constants.TusdAbiString)) + Expect(c.Name).To(Equal("TrueUSD")) + Expect(c.Address).To(Equal(tusdAddr)) }) It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() { - blockRetriever := &fakes.MockFullBlockRetriever{} - blockRetriever.FirstBlockErr = fakes.FakeError - t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) }) It("Does nothing if watched events are unset", func() { - t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Events = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).To(HaveOccurred()) - err := t.Init() - - Expect(err).ToNot(HaveOccurred()) - - _, ok := t.Contracts[fakeAddress] + _, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(false)) }) }) -}) -func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { - return transformer.Transformer{ - FilterRepository: &fakes.MockFilterRepository{}, - Parser: parsr, - BlockRetriever: blockRetriever, - Poller: pollr, - Contracts: map[string]*contract.Contract{}, - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - } -} + Describe("Execute", func() { + BeforeEach(func() { + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1) + blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.TransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log) + + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee")) + Expect(log.Block).To(Equal(int64(6194634))) + Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21")) + Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")) + Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146")) + }) + + It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(true)) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[""] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given methods using generated token holder address", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.BalanceOf{} + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("0")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("0")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("Fails if initialization has not been done", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + + err = t.Execute() + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Execute- against ENS registry contract", func() { + BeforeEach(func() { + blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1) + blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) + + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.NewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) + + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb")) + Expect(log.Block).To(Equal(int64(6194635))) + Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")) + Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")) + Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21")) + }) + + It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + c, ok := t.Contracts[ensAddr] + Expect(ok).To(Equal(true)) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EmittedHashes)).To(Equal(3)) + + b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + // Doesn't keep track of address since it wouldn't be used in calling the 'owner' method + _, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given methods using generated token holder address", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("It does not perist events if they do not pass the emitted arg filter", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log) + Expect(err).To(HaveOccurred()) + }) + + It("If a method arg filter is applied, only those arguments are used in polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/pkg/omni/light/converter/converter.go b/pkg/omni/light/converter/converter.go index b2c0d7ac..36c18f68 100644 --- a/pkg/omni/light/converter/converter.go +++ b/pkg/omni/light/converter/converter.go @@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) { // Convert the given watched event log into a types.Log for the given event func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) returnLogs := make([]types.Log, 0, len(logs)) for _, log := range logs { values := make(map[string]interface{}) @@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in values[field.Name] = i } - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } @@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) { - boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) + contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil) eventsToLogs := make(map[string][]types.Log) for _, event := range events { eventsToLogs[event.Name] = make([]types.Log, 0, len(logs)) @@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E // If the log is of this event type, process it as such if event.Sig() == log.Topics[0] { values := make(map[string]interface{}) - err := boundContract.UnpackLogIntoMap(values, event.Name, log) + err := contract.UnpackLogIntoMap(values, event.Name, log) if err != nil { return nil, err } diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/omni/light/repository/header_repository.go index 87d81990..d2aeac3d 100644 --- a/pkg/omni/light/repository/header_repository.go +++ b/pkg/omni/light/repository/header_repository.go @@ -17,8 +17,8 @@ package repository import ( + "database/sql" "fmt" - "github.com/jmoiron/sqlx" "github.com/hashicorp/golang-lru" @@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) } func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error { - tx, err := r.db.Beginx() + tx, err := r.db.Begin() if err != nil { return err } @@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) { return r.columns.Get(key) } -func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error { +func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error { _, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`) VALUES ($1, $2) ON CONFLICT (header_id) DO diff --git a/pkg/omni/light/retriever/retriever_suite_test.go b/pkg/omni/light/retriever/retriever_suite_test.go index 83a16269..5a254ef2 100644 --- a/pkg/omni/light/retriever/retriever_suite_test.go +++ b/pkg/omni/light/retriever/retriever_suite_test.go @@ -27,7 +27,7 @@ import ( func TestRetriever(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Light Block Number Retriever Suite Test") + RunSpecs(t, "Light BLock Number Retriever Suite Test") } var _ = BeforeSuite(func() { diff --git a/pkg/omni/light/transformer/transformer.go b/pkg/omni/light/transformer/transformer.go index f7738c0a..b0d4f37d 100644 --- a/pkg/omni/light/transformer/transformer.go +++ b/pkg/omni/light/transformer/transformer.go @@ -18,6 +18,7 @@ package transformer import ( "errors" + "github.com/vulcanize/vulcanizedb/pkg/config" "strings" "github.com/ethereum/go-ethereum/common" @@ -37,7 +38,7 @@ import ( ) // Requires a light synced vDB (headers) and a running eth node (or infura) -type Transformer struct { +type transformer struct { // Database interfaces srep.EventRepository // Holds transformed watched event log data repository.HeaderRepository // Interface for interaction with header repositories @@ -51,32 +52,12 @@ type Transformer struct { converter.Converter // Converts watched event logs into custom log poller.Poller // Polls methods using arguments collected from events and persists them using a method datastore - // Ethereum network name; default "" is mainnet - Network string + // Store contract configuration information + Config config.ContractConfig // Store contract info as mapping to contract address Contracts map[string]*contract.Contract - // Targeted subset of events/methods - // Stored as maps of contract address to events/method names of interest - WatchedEvents map[string][]string // Default/empty event list means all are watched - WantedMethods map[string][]string // Default/empty method list means none are polled - - // Starting block number for each contract - ContractStart map[string]int64 - - // Lists of argument values to filter event or - // method data with; if empty no filter is applied - EventArgs map[string][]string - MethodArgs map[string][]string - - // Whether or not to create a list of emitted address or hashes for the contract in postgres - CreateAddrList map[string]bool - CreateHashList map[string]bool - - // Method piping on/off for a contract - Piping map[string]bool - // Internally configured transformer variables contractAddresses []string // Holds all contract addresses, for batch fetching of logs sortedEventIds map[string][]string // Map to sort event column ids by contract, for post fetch processing and persisting of logs @@ -93,26 +74,18 @@ type Transformer struct { // 4. Execute // Transformer takes in config for blockchain, database, and network id -func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer { +func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *transformer { - return &Transformer{ + return &transformer{ Poller: poller.NewPoller(bc, db, types.LightSync), Fetcher: fetcher.NewFetcher(bc), - Parser: parser.NewParser(network), + Parser: parser.NewParser(con.Network), HeaderRepository: repository.NewHeaderRepository(db), BlockRetriever: retriever.NewBlockRetriever(db), Converter: converter.NewConverter(&contract.Contract{}), Contracts: map[string]*contract.Contract{}, EventRepository: srep.NewEventRepository(db, types.LightSync), - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - Piping: map[string]bool{}, - Network: network, + Config: con, } } @@ -120,7 +93,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transf // Loops over all of the addr => filter sets // Uses parser to pull event info from abi // Use this info to generate event filters -func (tr *Transformer) Init() error { +func (tr *transformer) Init() error { // Initialize internally configured transformer settings tr.contractAddresses = make([]string, 0) // Holds all contract addresses, for batch fetching of logs tr.sortedEventIds = make(map[string][]string) // Map to sort event column ids by contract, for post fetch processing and persisting of logs @@ -130,11 +103,20 @@ func (tr *Transformer) Init() error { tr.start = 100000000000 // Hold the lowest starting block and the highest ending block // Iterate through all internal contract addresses - for contractAddr, subset := range tr.WatchedEvents { - // Get Abi - err := tr.Parser.Parse(contractAddr) - if err != nil { - return err + for contractAddr := range tr.Config.Addresses { + // Configure Abi + if tr.Config.Abis[contractAddr] == "" { + // If no abi is given in the config, this method will try fetching from internal look-up table and etherscan + err := tr.Parser.Parse(contractAddr) + if err != nil { + return err + } + } else { + // If we have an abi from the config, load that into the parser + err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr]) + if err != nil { + return err + } } // Get first block and most recent block number in the header repo @@ -148,40 +130,38 @@ func (tr *Transformer) Init() error { } // Set to specified range if it falls within the bounds - if firstBlock < tr.ContractStart[contractAddr] { - firstBlock = tr.ContractStart[contractAddr] + if firstBlock < tr.Config.StartingBlocks[contractAddr] { + firstBlock = tr.Config.StartingBlocks[contractAddr] } // Get contract name if it has one var name = new(string) - tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock) + tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) - // Remove any potential accidental duplicate inputs in arg filter values + // Remove any potential accidental duplicate inputs eventArgs := map[string]bool{} - for _, arg := range tr.EventArgs[contractAddr] { + for _, arg := range tr.Config.EventArgs[contractAddr] { eventArgs[arg] = true } methodArgs := map[string]bool{} - for _, arg := range tr.MethodArgs[contractAddr] { + for _, arg := range tr.Config.MethodArgs[contractAddr] { methodArgs[arg] = true } // Aggregate info into contract object and store for execution con := contract.Contract{ - Name: *name, - Network: tr.Network, - Address: contractAddr, - Abi: tr.Parser.Abi(), - ParsedAbi: tr.Parser.ParsedAbi(), - StartingBlock: firstBlock, - LastBlock: -1, - Events: tr.Parser.GetEvents(subset), - Methods: tr.Parser.GetSelectMethods(tr.WantedMethods[contractAddr]), - FilterArgs: eventArgs, - MethodArgs: methodArgs, - CreateAddrList: tr.CreateAddrList[contractAddr], - CreateHashList: tr.CreateHashList[contractAddr], - Piping: tr.Piping[contractAddr], + Name: *name, + Network: tr.Config.Network, + Address: contractAddr, + Abi: tr.Parser.Abi(), + ParsedAbi: tr.Parser.ParsedAbi(), + StartingBlock: firstBlock, + LastBlock: -1, + Events: tr.Parser.GetEvents(tr.Config.Events[contractAddr]), + Methods: tr.Parser.GetSelectMethods(tr.Config.Methods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + Piping: tr.Config.Piping[contractAddr], }.Init() tr.Contracts[contractAddr] = con tr.contractAddresses = append(tr.contractAddresses, con.Address) @@ -221,7 +201,7 @@ func (tr *Transformer) Init() error { return nil } -func (tr *Transformer) Execute() error { +func (tr *transformer) Execute() error { if len(tr.Contracts) == 0 { return errors.New("error: transformer has no initialized contracts") } @@ -311,7 +291,7 @@ func (tr *Transformer) Execute() error { } // Used to poll contract methods at a given header -func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error { +func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error { for _, con := range tr.Contracts { // Skip method polling processes if no methods are specified // Also don't try to poll methods below this contract's specified starting block @@ -335,42 +315,6 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str return nil } -// Used to set which contract addresses and which of their events to watch -func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) { - tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to watch events for -func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) { - tr.EventArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set which contract addresses and which of their methods to call -func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) { - tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set subset of account addresses to poll methods on -func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) { - tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet -} - -// Used to set the block range to watch for a given address -func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) { - tr.ContractStart[strings.ToLower(contractAddr)] = start -} - -// Used to set whether or not to persist an account address list -func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) { - tr.CreateAddrList[strings.ToLower(contractAddr)] = on -} - -// Used to set whether or not to persist an hash list -func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) { - tr.CreateHashList[strings.ToLower(contractAddr)] = on -} - -// Used to turn method piping on for a contract -func (tr *Transformer) SetPiping(contractAddr string, on bool) { - tr.Piping[strings.ToLower(contractAddr)] = on +func (tr *transformer) GetConfig() config.ContractConfig { + return tr.Config } diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/omni/light/transformer/transformer_test.go index 31e3c194..093cd5f2 100644 --- a/pkg/omni/light/transformer/transformer_test.go +++ b/pkg/omni/light/transformer/transformer_test.go @@ -17,150 +17,446 @@ package transformer_test import ( + "fmt" + "github.com/vulcanize/vulcanizedb/pkg/config" + "strings" + + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/vulcanize/vulcanizedb/pkg/fakes" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" ) var _ = Describe("Transformer", func() { - var fakeAddress = "0x1234567890abcdef" + var db *postgres.DB + var err error + var blockChain core.BlockChain + var headerRepository repositories.HeaderRepository + var headerID, headerID2 int64 + var ensAddr = strings.ToLower(constants.EnsContractAddress) + var tusdAddr = strings.ToLower(constants.TusdContractAddress) - Describe("SetEvents", func() { - It("Sets which events to watch from the given contract address", func() { - watchedEvents := []string{"Transfer", "Mint"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, watchedEvents) - Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents)) - }) + BeforeEach(func() { + db, blockChain = test_helpers.SetupDBandBC() + headerRepository = repositories.NewHeaderRepository(db) }) - Describe("SetEventAddrs", func() { - It("Sets which account addresses to watch events for", func() { - eventAddrs := []string{"test1", "test2"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEventArgs(fakeAddress, eventAddrs) - Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs)) - }) - }) - - Describe("SetMethods", func() { - It("Sets which methods to poll at the given contract address", func() { - watchedMethods := []string{"balanceOf", "totalSupply"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethods(fakeAddress, watchedMethods) - Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods)) - }) - }) - - Describe("SetMethodAddrs", func() { - It("Sets which account addresses to poll methods against", func() { - methodAddrs := []string{"test1", "test2"} - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetMethodArgs(fakeAddress, methodAddrs) - Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs)) - }) - }) - - Describe("SetStartingBlock", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetStartingBlock(fakeAddress, 11) - Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11))) - }) - }) - - Describe("SetCreateAddrList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateAddrList(fakeAddress, true) - Expect(t.CreateAddrList[fakeAddress]).To(Equal(true)) - }) - }) - - Describe("SetCreateHashList", func() { - It("Sets the block range that the contract should be watched within", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetCreateHashList(fakeAddress, true) - Expect(t.CreateHashList[fakeAddress]).To(Equal(true)) - }) + AfterEach(func() { + test_helpers.TearDown(db) }) Describe("Init", func() { It("Initializes transformer's contract objects", func() { - blockRetriever := &fakes.MockLightBlockRetriever{} - firstBlock := int64(1) - blockRetriever.FirstBlock = firstBlock - - parsr := &fakes.MockParser{} - fakeAbi := "fake_abi" - parsr.AbiToReturn = fakeAbi - - pollr := &fakes.MockPoller{} - fakeContractName := "fake_contract_name" - pollr.ContractName = fakeContractName - - t := getFakeTransformer(blockRetriever, parsr, pollr) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).ToNot(HaveOccurred()) - c, ok := t.Contracts[fakeAddress] + c, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(true)) - Expect(c.StartingBlock).To(Equal(firstBlock)) + Expect(c.StartingBlock).To(Equal(int64(6194632))) Expect(c.LastBlock).To(Equal(int64(-1))) - Expect(c.Abi).To(Equal(fakeAbi)) - Expect(c.Name).To(Equal(fakeContractName)) - Expect(c.Address).To(Equal(fakeAddress)) + Expect(c.Abi).To(Equal(constants.TusdAbiString)) + Expect(c.Name).To(Equal("TrueUSD")) + Expect(c.Address).To(Equal(tusdAddr)) }) It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() { - blockRetriever := &fakes.MockLightBlockRetriever{} - blockRetriever.FirstBlockErr = fakes.FakeError - t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{}) - t.SetEvents(fakeAddress, []string{"Transfer"}) - - err := t.Init() - + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(fakes.FakeError)) }) - It("Does nothing if watched events are unset", func() { - t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{}) - - err := t.Init() - + It("Does nothing if nothing if no addresses are configured", func() { + headerRepository.CreateOrUpdateHeader(mocks.MockHeader1) + headerRepository.CreateOrUpdateHeader(mocks.MockHeader3) + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Addresses = nil + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() Expect(err).ToNot(HaveOccurred()) - _, ok := t.Contracts[fakeAddress] + _, ok := t.Contracts[tusdAddr] Expect(ok).To(Equal(false)) }) }) -}) + Describe("Execute- against TrueUSD contract", func() { + BeforeEach(func() { + header1, err := blockChain.GetHeaderByNumber(6791668) + Expect(err).ToNot(HaveOccurred()) + header2, err := blockChain.GetHeaderByNumber(6791669) + Expect(err).ToNot(HaveOccurred()) + header3, err := blockChain.GetHeaderByNumber(6791670) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header1) + headerID, err = headerRepository.CreateOrUpdateHeader(header2) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header3) + }) -func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer { - return transformer.Transformer{ - Parser: parsr, - BlockRetriever: blockRetriever, - Poller: pollr, - HeaderRepository: &fakes.MockLightHeaderRepository{}, - Contracts: map[string]*contract.Contract{}, - WatchedEvents: map[string][]string{}, - WantedMethods: map[string][]string{}, - ContractStart: map[string]int64{}, - EventArgs: map[string][]string{}, - MethodArgs: map[string][]string{}, - CreateAddrList: map[string]bool{}, - CreateHashList: map[string]bool{}, - } -} + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightTransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.HeaderID).To(Equal(headerID)) + Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) + Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) + Expect(log.Value).To(Equal("9998940000000000000000")) + }) + + It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + c, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(true)) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EmittedAddrs)).To(Equal(4)) + Expect(len(c.EmittedHashes)).To(Equal(0)) + + b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[""] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given methods using generated token holder address", func() { + var testConf config.ContractConfig + testConf = mocks.TusdConfig + testConf.Methods = map[string][]string{ + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.BalanceOf{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("55849938025000000000000")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("Fails if initialization has not been done", func() { + t := transformer.NewTransformer(mocks.TusdConfig, blockChain, db) + err = t.Execute() + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Execute- against ENS registry contract", func() { + BeforeEach(func() { + header1, err := blockChain.GetHeaderByNumber(6885695) + Expect(err).ToNot(HaveOccurred()) + header2, err := blockChain.GetHeaderByNumber(6885696) + Expect(err).ToNot(HaveOccurred()) + header3, err := blockChain.GetHeaderByNumber(6885697) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header1) + headerID, err = headerRepository.CreateOrUpdateHeader(header2) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header3) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.ENSConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(log.HeaderID).To(Equal(headerID)) + Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) + Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) + Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + }) + + It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + c, ok := t.Contracts[ensAddr] + Expect(ok).To(Equal(true)) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EmittedHashes)).To(Equal(2)) + Expect(len(c.EmittedAddrs)).To(Equal(0)) + + b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + // Doesn't keep track of address since it wouldn't be used in calling the 'owner' method + _, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")] + Expect(ok).To(Equal(false)) + }) + + It("Polls given method using list of collected hashes", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + + It("It does not persist events if they do not pass the emitted arg filter", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.EventArgs = map[string][]string{ + ensAddr: {"fake_filter_value"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + log := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log) + Expect(err).To(HaveOccurred()) + }) + + It("If a method arg filter is applied, only those arguments are used in polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + } + testConf.MethodArgs = map[string][]string{ + ensAddr: {"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + res := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + Expect(res.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Execute- against both ENS and TrueUSD", func() { + BeforeEach(func() { + header1, err := blockChain.GetHeaderByNumber(6791668) + Expect(err).ToNot(HaveOccurred()) + header2, err := blockChain.GetHeaderByNumber(6791669) + Expect(err).ToNot(HaveOccurred()) + header3, err := blockChain.GetHeaderByNumber(6791670) + Expect(err).ToNot(HaveOccurred()) + header4, err := blockChain.GetHeaderByNumber(6885695) + Expect(err).ToNot(HaveOccurred()) + header5, err := blockChain.GetHeaderByNumber(6885696) + Expect(err).ToNot(HaveOccurred()) + header6, err := blockChain.GetHeaderByNumber(6885697) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header1) + headerID, err = headerRepository.CreateOrUpdateHeader(header2) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header3) + headerRepository.CreateOrUpdateHeader(header4) + headerID2, err = headerRepository.CreateOrUpdateHeader(header5) + Expect(err).ToNot(HaveOccurred()) + headerRepository.CreateOrUpdateHeader(header6) + }) + + It("Transforms watched contract data into custom repositories", func() { + t := transformer.NewTransformer(mocks.ENSandTusdConfig, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + newOwnerLog := test_helpers.LightNewOwnerLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(newOwnerLog.HeaderID).To(Equal(headerID2)) + Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")) + Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")) + Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + + transferLog := test_helpers.LightTransferLog{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog) + Expect(err).ToNot(HaveOccurred()) + // We don't know vulcID, so compare individual fields instead of complete structures + Expect(transferLog.HeaderID).To(Equal(headerID)) + Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")) + Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")) + Expect(transferLog.Value).To(Equal("9998940000000000000000")) + }) + + It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() { + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + ens, ok := t.Contracts[ensAddr] + Expect(ok).To(Equal(true)) + tusd, ok := t.Contracts[tusdAddr] + Expect(ok).To(Equal(true)) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + Expect(len(ens.EmittedHashes)).To(Equal(2)) + Expect(len(ens.EmittedAddrs)).To(Equal(0)) + Expect(len(tusd.EmittedAddrs)).To(Equal(4)) + Expect(len(tusd.EmittedHashes)).To(Equal(0)) + + b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + }) + + It("Polls given methods for each contract, using list of collected values", func() { + var testConf config.ContractConfig + testConf = mocks.ENSandTusdConfig + testConf.Methods = map[string][]string{ + ensAddr: {"owner"}, + tusdAddr: {"balanceOf"}, + } + t := transformer.NewTransformer(testConf, blockChain, db) + err = t.Init() + Expect(err).ToNot(HaveOccurred()) + err = t.Execute() + Expect(err).ToNot(HaveOccurred()) + + owner := test_helpers.Owner{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner) + Expect(err).ToNot(HaveOccurred()) + Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")) + Expect(owner.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner) + Expect(err).ToNot(HaveOccurred()) + Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000")) + Expect(owner.TokenName).To(Equal("")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner) + Expect(err).To(HaveOccurred()) + + bal := test_helpers.BalanceOf{} + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal) + Expect(err).ToNot(HaveOccurred()) + Expect(bal.Balance).To(Equal("55849938025000000000000")) + Expect(bal.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/pkg/omni/shared/constants/constants.go b/pkg/omni/shared/constants/constants.go index 9ac4bd80..86012222 100644 --- a/pkg/omni/shared/constants/constants.go +++ b/pkg/omni/shared/constants/constants.go @@ -72,7 +72,6 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E" var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b" var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5" -// TODO: Consider whether these should be moved to plugins // Contract Owner var DaiContractOwner = "0x0000000000000000000000000000000000000000" var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f" diff --git a/pkg/omni/shared/contract/contract.go b/pkg/omni/shared/contract/contract.go index 19364d91..e9f06fb2 100644 --- a/pkg/omni/shared/contract/contract.go +++ b/pkg/omni/shared/contract/contract.go @@ -64,12 +64,6 @@ func (c Contract) Init() *Contract { } } - // If we are creating an address list in postgres - // we initialize the map despite what method call, if any - if c.CreateAddrList { - c.EmittedAddrs = map[interface{}]bool{} - } - return &c } diff --git a/pkg/omni/shared/helpers/test_helpers/database.go b/pkg/omni/shared/helpers/test_helpers/database.go index fb12cb3c..8a1620f7 100644 --- a/pkg/omni/shared/helpers/test_helpers/database.go +++ b/pkg/omni/shared/helpers/test_helpers/database.go @@ -106,7 +106,6 @@ type Owner struct { Address string `db:"returned"` } -// TODO: consider whether this should be moved to libraries/shared func SetupBC() core.BlockChain { infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" rawRpcClient, err := rpc.Dial(infuraIPC) @@ -114,9 +113,9 @@ func SetupBC() core.BlockChain { rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) ethClient := ethclient.NewClient(rawRpcClient) blockChainClient := client.NewEthClient(ethClient) - blockChainNode := node.MakeNode(rpcClient) + node := node.MakeNode(rpcClient) transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter) + blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) return blockChain } @@ -128,9 +127,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) { rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) ethClient := ethclient.NewClient(rawRpcClient) blockChainClient := client.NewEthClient(ethClient) - blockChainNode := node.MakeNode(rpcClient) + node := node.MakeNode(rpcClient) transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) - blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter) + blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter) db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -189,7 +188,6 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract }.Init() } -// TODO: consider whether this can be moved to plugin or libraries/shared func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) { db, err := postgres.NewDB(config.Database{ Hostname: "localhost", @@ -238,7 +236,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract { } func TearDown(db *postgres.DB) { - tx, err := db.Beginx() + tx, err := db.Begin() Expect(err).NotTo(HaveOccurred()) _, err = tx.Exec(`DELETE FROM blocks`) diff --git a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go b/pkg/omni/shared/helpers/test_helpers/mocks/entities.go index 9a2800c5..1829448b 100644 --- a/pkg/omni/shared/helpers/test_helpers/mocks/entities.go +++ b/pkg/omni/shared/helpers/test_helpers/mocks/entities.go @@ -18,6 +18,8 @@ package mocks import ( "encoding/json" + "github.com/vulcanize/vulcanizedb/pkg/config" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -249,3 +251,78 @@ var MockNewOwnerLog2 = types.Log{ }, Data: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000af21"), } + +var ens = strings.ToLower(constants.EnsContractAddress) +var tusd = strings.ToLower(constants.TusdContractAddress) + +var TusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + tusd: true, + }, + Abis: map[string]string{ + tusd: "", + }, + Events: map[string][]string{ + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + tusd: nil, + }, + MethodArgs: map[string][]string{ + tusd: nil, + }, + EventArgs: map[string][]string{ + tusd: nil, + }, +} + +var ENSConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + }, + Abis: map[string]string{ + ens: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + }, + Methods: map[string][]string{ + ens: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + }, +} + +var ENSandTusdConfig = config.ContractConfig{ + Network: "", + Addresses: map[string]bool{ + ens: true, + tusd: true, + }, + Abis: map[string]string{ + ens: "", + tusd: "", + }, + Events: map[string][]string{ + ens: []string{"NewOwner"}, + tusd: []string{"Transfer"}, + }, + Methods: map[string][]string{ + ens: nil, + tusd: nil, + }, + MethodArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, + EventArgs: map[string][]string{ + ens: nil, + tusd: nil, + }, +} diff --git a/pkg/omni/shared/repository/event_repository.go b/pkg/omni/shared/repository/event_repository.go index a08849b4..4be2a1fe 100644 --- a/pkg/omni/shared/repository/event_repository.go +++ b/pkg/omni/shared/repository/event_repository.go @@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c // Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB) func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - tx, err := r.db.Beginx() + tx, err := r.db.Begin() if err != nil { return err } @@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB) func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - tx, err := r.db.Beginx() + tx, err := r.db.Begin() if err != nil { return err } diff --git a/pkg/omni/shared/repository/method_repository.go b/pkg/omni/shared/repository/method_repository.go index e916e045..a55adbf5 100644 --- a/pkg/omni/shared/repository/method_repository.go +++ b/pkg/omni/shared/repository/method_repository.go @@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ // Creates a custom postgres command to persist logs for the given event func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { - tx, err := r.DB.Beginx() + tx, err := r.DB.Begin() if err != nil { return err } diff --git a/pkg/omni/shared/transformer/interface.go b/pkg/omni/shared/transformer/interface.go deleted file mode 100644 index ef0d331b..00000000 --- a/pkg/omni/shared/transformer/interface.go +++ /dev/null @@ -1,32 +0,0 @@ -// VulcanizeDB -// Copyright © 2019 Vulcanize - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package transformer - -// Used to extract any/all events and a subset of method (state variable) -// data for any contract and persists it to custom postgres tables in vDB -type Transformer interface { - SetEvents(contractAddr string, filterSet []string) - SetEventArgs(contractAddr string, filterSet []string) - SetMethods(contractAddr string, filterSet []string) - SetMethodArgs(contractAddr string, filterSet []string) - SetStartingBlock(contractAddr string, start int64) - SetCreateAddrList(contractAddr string, on bool) - SetCreateHashList(contractAddr string, on bool) - SetPiping(contractAddr string, on bool) - Init() error - Execute() error -}