From e9dbd771e5385d638760f41706361d2d7f056459 Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Sat, 3 Nov 2018 14:02:31 -0500 Subject: [PATCH] tests and fixes for fetcher, parser, retriever, converter, and repository; update cmd and watcher --- cmd/omniWatcher.go | 39 +++- cmd/root.go | 1 + libraries/shared/watcher.go | 4 + pkg/omni/converter/converter.go | 30 +-- pkg/omni/converter/converter_suite_test.go | 33 ++++ pkg/omni/converter/converter_test.go | 80 ++++++++ pkg/omni/fetcher/fetcher.go | 6 +- pkg/omni/fetcher/fetcher_suite_test.go | 33 ++++ pkg/omni/fetcher/fetcher_test.go | 145 ++++++++++++++ pkg/omni/parser/parser.go | 14 +- pkg/omni/parser/parser_suite_test.go | 33 ++++ pkg/omni/parser/parser_test.go | 62 ++++++ pkg/omni/repository/repository.go | 55 ++++-- pkg/omni/repository/repository_suite_test.go | 33 ++++ pkg/omni/repository/repository_test.go | 126 ++++++++++++ pkg/omni/retriever/retriever.go | 6 +- pkg/omni/retriever/retriever_suite_test.go | 33 ++++ pkg/omni/retriever/retriever_test.go | 181 ++++++++++++++++++ pkg/omni/transformer/event_transformer.go | 37 ++-- .../event_transformer_suite_test.go | 33 ++++ .../transformer/event_transformer_test.go | 15 ++ pkg/omni/transformer/integration_test.go | 15 ++ pkg/omni/types/entities.go | 16 +- .../go-ethereum/accounts/abi/argument.go | 3 +- 24 files changed, 958 insertions(+), 75 deletions(-) create mode 100644 pkg/omni/converter/converter_suite_test.go create mode 100644 pkg/omni/converter/converter_test.go create mode 100644 pkg/omni/fetcher/fetcher_suite_test.go create mode 100644 pkg/omni/fetcher/fetcher_test.go create mode 100644 pkg/omni/parser/parser_suite_test.go create mode 100644 pkg/omni/parser/parser_test.go create mode 100644 pkg/omni/repository/repository_suite_test.go create mode 100644 pkg/omni/repository/repository_test.go create mode 100644 pkg/omni/retriever/retriever_suite_test.go create mode 100644 pkg/omni/retriever/retriever_test.go create mode 100644 pkg/omni/transformer/event_transformer_suite_test.go create mode 100644 pkg/omni/transformer/event_transformer_test.go create mode 100644 pkg/omni/transformer/integration_test.go diff --git a/cmd/omniWatcher.go b/cmd/omniWatcher.go index 59de122e..2b08a4a1 100644 --- a/cmd/omniWatcher.go +++ b/cmd/omniWatcher.go @@ -20,11 +20,13 @@ import ( "log" "os" "strings" + "time" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/spf13/cobra" + "github.com/vulcanize/vulcanizedb/libraries/shared" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/geth" "github.com/vulcanize/vulcanizedb/pkg/geth/client" @@ -58,16 +60,24 @@ Requires a .toml config file: } func omniWatcher() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() - if contractAddress == "" { + if contractAddress == "" && len(contractAddresses) == 0 { log.Fatal("Contract address required") } - if contractEvents == nil { + if len(contractEvents) == 0 || len(contractMethods) == 0 { var str string for str != "y" { reader := bufio.NewReader(os.Stdin) - fmt.Print("Warning: no events specified, proceeding to watch every event at address" + contractAddress + "? (Y/n)\n> ") + if len(contractEvents) == 0 && len(contractMethods) == 0 { + fmt.Print("Warning: no events or methods specified, proceed to watch every event and method? (Y/n)\n> ") + } else if len(contractEvents) == 0 { + fmt.Print("Warning: no events specified, proceed to watch every event? (Y/n)\n> ") + } else { + fmt.Print("Warning: no methods specified, proceed to watch every method? (Y/n)\n> ") + } resp, err := reader.ReadBytes('\n') if err != nil { log.Fatal(err) @@ -82,8 +92,9 @@ func omniWatcher() { rawRpcClient, err := rpc.Dial(ipc) if err != nil { - log.Fatal(err) + log.Fatal(fmt.Sprintf("Failed to initialize rpc client\r\nerr: %v\r\n", err)) } + rpcClient := client.NewRpcClient(rawRpcClient, ipc) ethClient := ethclient.NewClient(rawRpcClient) client := client.NewEthClient(ethClient) @@ -102,21 +113,31 @@ func omniWatcher() { } t := transformer.NewTransformer(&con) - t.Set(contractAddress, contractEvents) + + contractAddresses = append(contractAddresses, contractAddress) + for _, addr := range contractAddresses { + t.Set(addr, contractEvents) + } err = t.Init() if err != nil { - log.Fatal(fmt.Sprintf("Failed to initialized generator\r\nerr: %v\r\n", err)) + log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err)) } - log.Fatal(t.Execute()) + w := shared.Watcher{} + w.AddTransformer(t) + + for range ticker.C { + w.Execute() + } } func init() { rootCmd.AddCommand(omniWatcherCmd) omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") - omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch- use only with single address") - omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "Addresses of the contracts to generate watchers for") + omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "List of addresses to generate watchers for") + omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched") + omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-methods", "m", []string{}, "Subset of methods to watch; by default all methods are watched") omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) } diff --git a/cmd/root.go b/cmd/root.go index 20cf32a8..a062fada 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,6 +36,7 @@ var ( contractAddress string contractAddresses []string contractEvents []string + contractMethods []string ) var rootCmd = &cobra.Command{ diff --git a/libraries/shared/watcher.go b/libraries/shared/watcher.go index ca3f0a00..ed2ad4a0 100644 --- a/libraries/shared/watcher.go +++ b/libraries/shared/watcher.go @@ -23,6 +23,10 @@ func (watcher *Watcher) AddTransformers(us []TransformerInitializer, con Contrac return nil } +func (watcher *Watcher) AddTransformer(t Transformer) { + watcher.Transformers = append(watcher.Transformers, t) +} + func (watcher *Watcher) Execute() error { var err error for _, transformer := range watcher.Transformers { diff --git a/pkg/omni/converter/converter.go b/pkg/omni/converter/converter.go index 5bf169a9..b1afa3ba 100644 --- a/pkg/omni/converter/converter.go +++ b/pkg/omni/converter/converter.go @@ -24,6 +24,8 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/omni/types" ) +// Converter is used to convert watched event logs to +// custom logs containing event input name => value maps type Converter interface { Convert(watchedEvent core.WatchedEvent, event *types.Event) error Update(info types.ContractInfo) @@ -40,11 +42,12 @@ func NewConverter(info types.ContractInfo) *converter { } } -func (c converter) Update(info types.ContractInfo) { +func (c *converter) Update(info types.ContractInfo) { c.contractInfo = info } -func (c converter) Convert(watchedEvent core.WatchedEvent, event *types.Event) error { +// Convert the given watched event log into a types.Log for the given event +func (c *converter) Convert(watchedEvent core.WatchedEvent, event *types.Event) error { contract := bind.NewBoundContract(common.HexToAddress(c.contractInfo.Address), c.contractInfo.ParsedAbi, nil, nil, nil) values := make(map[string]interface{}) @@ -53,27 +56,24 @@ func (c converter) Convert(watchedEvent core.WatchedEvent, event *types.Event) e values[field.Name] = i switch field.Type.T { - case abi.StringTy: - field.PgType = "CHARACTER VARYING(66) NOT NULL" + case abi.StringTy, abi.HashTy, abi.AddressTy: + field.PgType = "CHARACTER VARYING(66)" case abi.IntTy, abi.UintTy: - field.PgType = "DECIMAL NOT NULL" + field.PgType = "DECIMAL" case abi.BoolTy: - field.PgType = "BOOLEAN NOT NULL" + field.PgType = "BOOLEAN" case abi.BytesTy, abi.FixedBytesTy: - field.PgType = "BYTEA NOT NULL" - case abi.AddressTy: - field.PgType = "CHARACTER VARYING(66) NOT NULL" - case abi.HashTy: - field.PgType = "CHARACTER VARYING(66) NOT NULL" + field.PgType = "BYTEA" case abi.ArrayTy: - field.PgType = "TEXT[] NOT NULL" + field.PgType = "TEXT[]" case abi.FixedPointTy: - field.PgType = "MONEY NOT NULL" // use shopspring/decimal for fixed point numbers in go and money type in postgres? + field.PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres? case abi.FunctionTy: - field.PgType = "TEXT NOT NULL" + field.PgType = "TEXT" default: - field.PgType = "TEXT NOT NULL" + field.PgType = "TEXT" } + } log := helpers.ConvertToLog(watchedEvent) diff --git a/pkg/omni/converter/converter_suite_test.go b/pkg/omni/converter/converter_suite_test.go new file mode 100644 index 00000000..b032ea91 --- /dev/null +++ b/pkg/omni/converter/converter_suite_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package converter_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestConverter(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Converter Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/converter/converter_test.go b/pkg/omni/converter/converter_test.go new file mode 100644 index 00000000..f14aaa4a --- /dev/null +++ b/pkg/omni/converter/converter_test.go @@ -0,0 +1,80 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package converter_test + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/examples/generic/helpers" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/omni/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/parser" + "github.com/vulcanize/vulcanizedb/pkg/omni/types" +) + +var mockEvent = core.WatchedEvent{ + LogID: 1, + Name: constants.TransferEvent.String(), + BlockNumber: 5488076, + Address: constants.TusdContractAddress, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Index: 110, + Topic0: constants.TransferEvent.Signature(), + Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21", + Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + Topic3: "", + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", +} + +var _ = Describe("Converter Test", func() { + + It("Converts watched event log to mapping of event input names to values", func() { + p := parser.NewParser("") + err := p.Parse(constants.TusdContractAddress) + Expect(err).ToNot(HaveOccurred()) + + info := types.ContractInfo{ + Name: "TrueUSD", + Address: constants.TusdContractAddress, + Abi: p.Abi(), + ParsedAbi: p.ParsedAbi(), + StartingBlock: 5197514, + Events: p.GetEvents(), + Methods: p.GetMethods(), + } + + event := info.Events["Transfer"] + + info.GenerateFilters([]string{"Transfer"}) + c := converter.NewConverter(info) + err = c.Convert(mockEvent, event) + Expect(err).ToNot(HaveOccurred()) + + from := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21") + to := common.HexToAddress("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391") + value := helpers.BigFromString("1097077688018008265106216665536940668749033598146") + + v := event.Logs[1].Values["value"].(*big.Int) + + Expect(event.Logs[1].Values["to"].(common.Address)).To(Equal(to)) + Expect(event.Logs[1].Values["from"].(common.Address)).To(Equal(from)) + Expect(v.String()).To(Equal(value.String())) + }) +}) diff --git a/pkg/omni/fetcher/fetcher.go b/pkg/omni/fetcher/fetcher.go index 1ad7fccc..e2197e71 100644 --- a/pkg/omni/fetcher/fetcher.go +++ b/pkg/omni/fetcher/fetcher.go @@ -26,8 +26,7 @@ import ( // Fetcher serves as the lower level data fetcher that calls the underlying // blockchain's FetchConctractData method for a given return type - -// Interface definition for a Fetcher +// Used to collect data from contract public methods type Fetcher interface { FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) @@ -36,12 +35,10 @@ type Fetcher interface { FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) } -// Fetcher struct type fetcher struct { BlockChain core.BlockChain // Underyling Blockchain } -// Fetcher error type fetcherError struct { err string fetchMethod string @@ -53,7 +50,6 @@ func NewFetcher(blockChain core.BlockChain) *fetcher { } } -// Fetcher error method func (fe *fetcherError) Error() string { return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err) } diff --git a/pkg/omni/fetcher/fetcher_suite_test.go b/pkg/omni/fetcher/fetcher_suite_test.go new file mode 100644 index 00000000..ac8e9e59 --- /dev/null +++ b/pkg/omni/fetcher/fetcher_suite_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fetcher_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestFetcher(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Fetcher Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/fetcher/fetcher_test.go b/pkg/omni/fetcher/fetcher_test.go new file mode 100644 index 00000000..975668fd --- /dev/null +++ b/pkg/omni/fetcher/fetcher_test.go @@ -0,0 +1,145 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fetcher_test + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/vulcanize/vulcanizedb/pkg/omni/fetcher" + "math/big" + + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" + rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc" + "github.com/vulcanize/vulcanizedb/pkg/geth/node" +) + +var _ = Describe("Fetcher Test", func() { + blockNumber := int64(6194634) + var realFetcher fetcher.Fetcher + + BeforeEach(func() { + infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05" + rawRpcClient, err := rpc.Dial(infuraIPC) + Expect(err).NotTo(HaveOccurred()) + rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC) + ethClient := ethclient.NewClient(rawRpcClient) + blockChainClient := client.NewEthClient(ethClient) + node := node.MakeNode(rpcClient) + transactionConverter := rpc2.NewRpcTransactionConverter(ethClient) + blockChain := geth.NewBlockChain(blockChainClient, node, transactionConverter) + realFetcher = fetcher.NewFetcher(blockChain) + }) + + Describe("Fetch big.Int test", func() { + + It("fetch totalSupply big.Int", func() { + bigInt, err := realFetcher.FetchBigInt("totalSupply", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil) + Expect(err).NotTo(HaveOccurred()) + expectedBigInt := big.Int{} + expectedBigInt.SetString("47327413946297204537985606", 10) + Expect(bigInt.String()).To(Equal(expectedBigInt.String())) + }) + + It("returns an error if the call to the blockchain fails", func() { + result, err := realFetcher.FetchBigInt("totalSupply", "", "", 0, nil) + + Expect(result).To(Equal(big.Int{})) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("totalSupply")) + }) + }) + + Describe("Fetch bool test", func() { + + It("fetch stopped boolean", func() { + boo, err := realFetcher.FetchBool("stopped", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(boo).To(Equal(false)) + }) + + It("returns an error if the call to the blockchain fails", func() { + boo, err := realFetcher.FetchBool("stopped", "", "", 0, nil) + + Expect(boo).To(Equal(false)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("stopped")) + }) + }) + + Describe("Fetch address test", func() { + + It("fetch owner address", func() { + expectedAddr := common.HexToAddress("0x0000000000000000000000000000000000000000") + addr, err := realFetcher.FetchAddress("owner", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(addr).To(Equal(expectedAddr)) + }) + + It("returns an error if the call to the blockchain fails", func() { + addr, err := realFetcher.FetchAddress("owner", "", "", 0, nil) + + Expect(addr).To(Equal(common.Address{})) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("owner")) + }) + }) + + Describe("Fetch string test", func() { + + It("fetch nae string", func() { + expectedStr := "TrueUSD" + str, err := realFetcher.FetchString("name", constants.TusdAbiString, constants.TusdContractAddress, blockNumber, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(str).To(Equal(expectedStr)) + }) + + It("returns an error if the call to the blockchain fails", func() { + str, err := realFetcher.FetchString("name", "", "", 0, nil) + + Expect(str).To(Equal("")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("name")) + }) + }) + + Describe("Fetch hash test", func() { + + It("fetch name hash", func() { + expectedHash := common.HexToHash("0x44616920537461626c65636f696e2076312e3000000000000000000000000000") + hash, err := realFetcher.FetchHash("name", constants.DaiAbiString, constants.DaiContractAddress, blockNumber, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(hash).To(Equal(expectedHash)) + }) + + It("returns an error if the call to the blockchain fails", func() { + hash, err := realFetcher.FetchHash("name", "", "", 0, nil) + + Expect(hash).To(Equal(common.Hash{})) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("name")) + }) + }) + +}) diff --git a/pkg/omni/parser/parser.go b/pkg/omni/parser/parser.go index 485bb3fa..7870cbfc 100644 --- a/pkg/omni/parser/parser.go +++ b/pkg/omni/parser/parser.go @@ -20,10 +20,12 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/omni/types" ) +// Parser is used to fetch and parse contract ABIs +// It is dependent on etherscan's api type Parser interface { Parse(contractAddr string) error - GetAbi() string - GetParsedAbi() abi.ABI + Abi() string + ParsedAbi() abi.ABI GetMethods() map[string]*types.Method GetEvents() map[string]*types.Event } @@ -42,14 +44,16 @@ func NewParser(network string) *parser { } } -func (p *parser) GetAbi() string { +func (p *parser) Abi() string { return p.abi } -func (p *parser) GetParsedAbi() abi.ABI { +func (p *parser) ParsedAbi() abi.ABI { return p.parsedAbi } +// Retrieves and parses the abi string +// the given contract address func (p *parser) Parse(contractAddr string) error { abiStr, err := p.client.GetAbi(contractAddr) if err != nil { @@ -62,6 +66,7 @@ func (p *parser) Parse(contractAddr string) error { return err } +// Parses methods into our custom method type and returns func (p *parser) GetMethods() map[string]*types.Method { methods := map[string]*types.Method{} @@ -73,6 +78,7 @@ func (p *parser) GetMethods() map[string]*types.Method { return methods } +// Parses events into our custom event type and returns func (p *parser) GetEvents() map[string]*types.Event { events := map[string]*types.Event{} diff --git a/pkg/omni/parser/parser_suite_test.go b/pkg/omni/parser/parser_suite_test.go new file mode 100644 index 00000000..35c14be0 --- /dev/null +++ b/pkg/omni/parser/parser_suite_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestParser(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Parser Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/parser/parser_test.go b/pkg/omni/parser/parser_test.go new file mode 100644 index 00000000..f7683560 --- /dev/null +++ b/pkg/omni/parser/parser_test.go @@ -0,0 +1,62 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/omni/parser" +) + +var _ = Describe("Parser Test", func() { + + var p parser.Parser + var err error + + BeforeEach(func() { + p = parser.NewParser("") + }) + + It("Fetches and parses abi using contract address", func() { + contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" // dai contract address + err = p.Parse(contractAddr) + Expect(err).ToNot(HaveOccurred()) + + expectedAbi := constants.DaiAbiString + Expect(p.Abi()).To(Equal(expectedAbi)) + + expectedParsedAbi, err := geth.ParseAbi(expectedAbi) + Expect(err).ToNot(HaveOccurred()) + Expect(p.ParsedAbi()).To(Equal(expectedParsedAbi)) + }) + + It("Returns parsed methods and events", func() { + contractAddr := "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" + err = p.Parse(contractAddr) + Expect(err).ToNot(HaveOccurred()) + + methods := p.GetMethods() + events := p.GetEvents() + + _, ok := methods["totalSupply"] + Expect(ok).To(Equal(true)) + + _, ok = events["Transfer"] + Expect(ok).To(Equal(true)) + }) + +}) diff --git a/pkg/omni/repository/repository.go b/pkg/omni/repository/repository.go index 5f9c023c..37cad02e 100644 --- a/pkg/omni/repository/repository.go +++ b/pkg/omni/repository/repository.go @@ -16,12 +16,15 @@ package repository import ( "fmt" + "github.com/ethereum/go-ethereum/common" + "math/big" "strings" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/omni/types" ) +// Repository is used to type DataStore interface { PersistEvents(info types.ContractInfo) error } @@ -36,6 +39,9 @@ func NewDataStore(db *postgres.DB) *dataStore { } } +// Creates a schema for the contract +// Creates tables for the watched contract events +// Persists converted event log data into these custom tables func (d *dataStore) PersistEvents(contract types.ContractInfo) error { schemaExists, err := d.CheckForSchema(contract.Name) @@ -50,7 +56,9 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error { } } - for eventName, event := range contract.Events { + for eventName := range contract.Filters { + + event := contract.Events[eventName] tableExists, err := d.CheckForTable(contract.Name, eventName) if err != nil { @@ -79,12 +87,31 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error { counter := 0 for inputName, input := range log.Values { + // postgres cannot handle custom types, resolve to strings + switch input.(type) { + case big.Int: + var b big.Int + b = input.(big.Int) + input = b.String() + case *big.Int: + var b *big.Int + b = input.(*big.Int) + input = b.String() + case common.Address: + var a common.Address + a = input.(common.Address) + input = a.String() + case common.Hash: + var h common.Hash + h = input.(common.Hash) + input = h.String() + } + counter += 1 - pgStr = pgStr + fmt.Sprintf(", %s", strings.ToLower(inputName)) + pgStr = pgStr + fmt.Sprintf(", _%s", strings.ToLower(inputName)) data = append(data, input) } - pgStr = pgStr + ") " appendStr := "VALUES ($1, $2, $3, $4, $5, $6" for i := 0; i < counter; i++ { @@ -100,51 +127,47 @@ func (d *dataStore) PersistEvents(contract types.ContractInfo) error { return err } } - } return nil } +// Creates a table for the given contract and event func (d *dataStore) CreateEventTable(contractName string, event *types.Event) error { // Create postgres command to create table for any given event pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s.%s ", strings.ToLower(contractName), strings.ToLower(event.Name)) - pgStr = pgStr + `(id SERIAL, - vulcanize_log_id INTEGER NOT NULL UNIQUE, - token_name CHARACTER VARYING(66) NOT NULL, - token_address CHARACTER VARYING(66) NOT NULL, - event_name CHARACTER VARYING(66) NOT NULL, - block INTEGER NOT NULL, - tx CHARACTER VARYING(66) NOT NULL, ` + pgStr = pgStr + "(id SERIAL, vulcanize_log_id INTEGER NOT NULL UNIQUE, token_name CHARACTER VARYING(66) NOT NULL, token_address CHARACTER VARYING(66) NOT NULL, event_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL, tx CHARACTER VARYING(66) NOT NULL," for _, field := range event.Fields { - pgStr = pgStr + fmt.Sprintf("%s %s NOT NULL, ", field.Name, field.PgType) + pgStr = pgStr + fmt.Sprintf(" _%s %s NOT NULL,", field.Name, field.PgType) } - pgStr = pgStr + "CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)" - + pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)" _, err := d.DB.Exec(pgStr) return err } +// Checks if a table already exists for the given contract and event func (d *dataStore) CheckForTable(contractName string, eventName string) (bool, error) { pgStr := fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '%s' AND table_name = '%s')", contractName, eventName) var exists bool - err := d.DB.Get(exists, pgStr) + err := d.DB.Get(&exists, pgStr) return exists, err } +// Creates a schema for the given contract func (d *dataStore) CreateContractSchema(contractName string) error { _, err := d.DB.Exec("CREATE SCHEMA IF NOT EXISTS " + contractName) return err } +// Checks if a schema already exists for the given contract func (d *dataStore) CheckForSchema(contractName string) (bool, error) { pgStr := fmt.Sprintf("SELECT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s')", contractName) var exists bool - err := d.DB.Get(exists, pgStr) + err := d.DB.Get(&exists, pgStr) return exists, err } diff --git a/pkg/omni/repository/repository_suite_test.go b/pkg/omni/repository/repository_suite_test.go new file mode 100644 index 00000000..a4102f55 --- /dev/null +++ b/pkg/omni/repository/repository_suite_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestRepository(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Repository Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/repository/repository_test.go b/pkg/omni/repository/repository_test.go new file mode 100644 index 00000000..5d392dca --- /dev/null +++ b/pkg/omni/repository/repository_test.go @@ -0,0 +1,126 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository_test + +import ( + "math/rand" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/examples/test_helpers" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/omni/converter" + "github.com/vulcanize/vulcanizedb/pkg/omni/parser" + "github.com/vulcanize/vulcanizedb/pkg/omni/repository" + "github.com/vulcanize/vulcanizedb/pkg/omni/types" +) + +var mockEvent = core.WatchedEvent{ + Name: constants.TransferEvent.String(), + BlockNumber: 5488076, + Address: constants.TusdContractAddress, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Index: 110, + Topic0: constants.TransferEvent.Signature(), + Topic1: "0x000000000000000000000000000000000000000000000000000000000000af21", + Topic2: "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + Topic3: "", + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", +} + +var _ = Describe("Converter Test", func() { + + var db *postgres.DB + var logRepository repositories.LogRepository + var blockRepository repositories.BlockRepository + var receiptRepository repositories.ReceiptRepository + var dataStore repository.DataStore + var err error + var info types.ContractInfo + var blockNumber int64 + var blockId int64 + var vulcanizeLogId int64 + rand.Seed(time.Now().UnixNano()) + + BeforeEach(func() { + db, err = postgres.NewDB(config.Database{ + Hostname: "localhost", + Name: "vulcanize_private", + Port: 5432, + }, core.Node{}) + Expect(err).NotTo(HaveOccurred()) + + receiptRepository = repositories.ReceiptRepository{DB: db} + logRepository = repositories.LogRepository{DB: db} + blockRepository = *repositories.NewBlockRepository(db) + + blockNumber = rand.Int63() + blockId = test_helpers.CreateBlock(blockNumber, blockRepository) + + log := core.Log{} + logs := []core.Log{log} + receipt := core.Receipt{ + Logs: logs, + } + receipts := []core.Receipt{receipt} + + err = receiptRepository.CreateReceiptsAndLogs(blockId, receipts) + Expect(err).ToNot(HaveOccurred()) + + err = logRepository.Get(&vulcanizeLogId, `SELECT id FROM logs`) + Expect(err).ToNot(HaveOccurred()) + + p := parser.NewParser("") + err = p.Parse(constants.TusdContractAddress) + Expect(err).ToNot(HaveOccurred()) + + info = types.ContractInfo{ + Name: "TrueUSD", + Address: constants.TusdContractAddress, + Abi: p.Abi(), + ParsedAbi: p.ParsedAbi(), + StartingBlock: 5197514, + Events: p.GetEvents(), + Methods: p.GetMethods(), + } + + event := info.Events["Transfer"] + info.GenerateFilters([]string{"Transfer"}) + c := converter.NewConverter(info) + mockEvent.LogID = vulcanizeLogId + err = c.Convert(mockEvent, event) + Expect(err).ToNot(HaveOccurred()) + + dataStore = repository.NewDataStore(db) + }) + + AfterEach(func() { + db.Query(`DELETE FROM blocks`) + db.Query(`DELETE FROM logs`) + db.Query(`DELETE FROM receipts`) + db.Query(`DROP SCHEMA IF EXISTS trueusd CASCADE`) + }) + + It("Convert watched event test", func() { + err = dataStore.PersistEvents(info) + Expect(err).ToNot(HaveOccurred()) + }) +}) diff --git a/pkg/omni/retriever/retriever.go b/pkg/omni/retriever/retriever.go index d4f9a0e6..0483ba29 100644 --- a/pkg/omni/retriever/retriever.go +++ b/pkg/omni/retriever/retriever.go @@ -18,6 +18,8 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" ) +// Retriever is used to retrieve the first block for a given contract +// It requires a vDB synced database with blocks, transactions, receipts, and logs type Retriever interface { RetrieveFirstBlock(contractAddr string) (int64, error) RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) @@ -35,7 +37,7 @@ func NewRetriever(db *postgres.DB) (r *retriever) { } } -// For some contracts the creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) +// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai) func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64, error) { var firstBlock int err := r.DB.Get( @@ -51,7 +53,7 @@ func (r *retriever) RetrieveFirstBlockFromReceipts(contractAddr string) (int64, return int64(firstBlock), err } -// This servers as a heuristic to find the first block by finding the first contract event log +// In which case this servers as a heuristic to find the first block by finding the first contract event log func (r *retriever) RetrieveFirstBlockFromLogs(contractAddr string) (int64, error) { var firstBlock int err := r.DB.Get( diff --git a/pkg/omni/retriever/retriever_suite_test.go b/pkg/omni/retriever/retriever_suite_test.go new file mode 100644 index 00000000..aef54ed2 --- /dev/null +++ b/pkg/omni/retriever/retriever_suite_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retriever_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestRetriever(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Retriever Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/retriever/retriever_test.go b/pkg/omni/retriever/retriever_test.go new file mode 100644 index 00000000..6d27463e --- /dev/null +++ b/pkg/omni/retriever/retriever_test.go @@ -0,0 +1,181 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retriever_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/examples/constants" + "github.com/vulcanize/vulcanizedb/pkg/config" + "github.com/vulcanize/vulcanizedb/pkg/core" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" + "github.com/vulcanize/vulcanizedb/pkg/omni/retriever" +) + +var _ = Describe("Fetcher Test", func() { + var db *postgres.DB + var r retriever.Retriever + var blockRepository repositories.BlockRepository + + BeforeEach(func() { + var err error + db, err = postgres.NewDB(config.Database{ + Hostname: "localhost", + Name: "vulcanize_private", + Port: 5432, + }, core.Node{}) + Expect(err).NotTo(HaveOccurred()) + + blockRepository = *repositories.NewBlockRepository(db) + + r = retriever.NewRetriever(db) + }) + + AfterEach(func() { + db.Query(`DELETE FROM blocks`) + db.Query(`DELETE FROM logs`) + db.Query(`DELETE FROM receipts`) + }) + + It("Retrieve first block number of a contract from receipt if possible", func() { + log := core.Log{ + BlockNumber: 2, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + Address: constants.TusdContractAddress, + Topics: core.Topics{ + constants.TransferEvent.Signature(), + "0x000000000000000000000000000000000000000000000000000000000000af21", + "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + "", + }, + Index: 1, + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", + } + + receipt1 := core.Receipt{ + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + ContractAddress: constants.TusdContractAddress, + Logs: []core.Log{}, + } + + receipt2 := core.Receipt{ + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + ContractAddress: constants.TusdContractAddress, + Logs: []core.Log{log}, + } + + transaction1 := core.Transaction{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Receipt: receipt1, + } + + transaction2 := core.Transaction{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + Receipt: receipt2, + } + + block1 := core.Block{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", + Number: 1, + Transactions: []core.Transaction{transaction1}, + } + + block2 := core.Block{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", + Number: 2, + Transactions: []core.Transaction{transaction2}, + } + + blockRepository.CreateOrUpdateBlock(block1) + blockRepository.CreateOrUpdateBlock(block2) + + i, err := r.RetrieveFirstBlock(constants.TusdContractAddress) + Expect(err).NotTo(HaveOccurred()) + Expect(i).To(Equal(int64(1))) + }) + + It("Retrieves first block number of a contract from event logs", func() { + log1 := core.Log{ + BlockNumber: 1, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Address: constants.DaiContractAddress, + Topics: core.Topics{ + constants.TransferEvent.Signature(), + "0x000000000000000000000000000000000000000000000000000000000000af21", + "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + "", + }, + Index: 1, + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", + } + + log2 := core.Log{ + BlockNumber: 2, + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + Address: constants.DaiContractAddress, + Topics: core.Topics{ + constants.TransferEvent.Signature(), + "0x000000000000000000000000000000000000000000000000000000000000af21", + "0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391", + "", + }, + Index: 1, + Data: "0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc200000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359000000000000000000000000000000000000000000000000392d2e2bda9c00000000000000000000000000000000000000000000000000927f41fa0a4a418000000000000000000000000000000000000000000000000000000000005adcfebe", + } + + receipt1 := core.Receipt{ + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + ContractAddress: "", + Logs: []core.Log{log1}, + } + + receipt2 := core.Receipt{ + TxHash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + ContractAddress: "", + Logs: []core.Log{log2}, + } + + transaction1 := core.Transaction{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad6546ae", + Receipt: receipt1, + } + + transaction2 := core.Transaction{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad234hfs", + Receipt: receipt2, + } + + block1 := core.Block{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad123ert", + Number: 1, + Transactions: []core.Transaction{transaction1}, + } + + block2 := core.Block{ + Hash: "0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad456yui", + Number: 2, + Transactions: []core.Transaction{transaction2}, + } + + blockRepository.CreateOrUpdateBlock(block1) + blockRepository.CreateOrUpdateBlock(block2) + + i, err := r.RetrieveFirstBlock(constants.DaiContractAddress) + Expect(err).NotTo(HaveOccurred()) + Expect(i).To(Equal(int64(1))) + }) +}) diff --git a/pkg/omni/transformer/event_transformer.go b/pkg/omni/transformer/event_transformer.go index c97b2194..c4fd3857 100644 --- a/pkg/omni/transformer/event_transformer.go +++ b/pkg/omni/transformer/event_transformer.go @@ -29,10 +29,11 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/omni/types" ) -// Top-level object similar to generator -// but attempts to solve problem without -// automated code generation - +// Omni event transformer +// Used to extract all or a subset of event data for +// any contract and persist it to postgres in a manner +// that requires no prior knowledge of the contract +// other than its address and which network it is on type EventTransformer interface { Init(contractAddr string) error } @@ -49,13 +50,14 @@ type eventTransformer struct { // Underlying interfaces parser.Parser // Parses events out of contract abi fetched with addr retriever.Retriever // Retrieves first block with contract addr referenced - fetcher.Fetcher // Fetches data from contract methods + fetcher.Fetcher // Fetches data from public contract methods + converter.Converter // Converts watched event logs into custom log // Store contract info as mapping to contract address ContractInfo map[string]types.ContractInfo // Subset of events of interest, stored as map of contract address to events - // By default this + // Default/empty list means all events are considered for that address sets map[string][]string } @@ -64,6 +66,7 @@ func NewTransformer(c *types.Config) (t *eventTransformer) { t.Parser = parser.NewParser(c.Network) t.Retriever = retriever.NewRetriever(c.DB) t.Fetcher = fetcher.NewFetcher(c.BC) + t.Converter = converter.NewConverter(types.ContractInfo{}) t.ContractInfo = map[string]types.ContractInfo{} t.WatchedEventRepository = repositories.WatchedEventRepository{DB: c.DB} t.FilterRepository = repositories.FilterRepository{DB: c.DB} @@ -91,11 +94,11 @@ func (t *eventTransformer) Init() error { } var ctrName string - strName, err1 := t.Fetcher.FetchString("name", t.Parser.GetAbi(), contractAddr, -1, nil) - if err1 != nil || strName == "" { - hashName, err2 := t.Fetcher.FetchHash("name", t.Parser.GetAbi(), contractAddr, -1, nil) - if err2 != nil || hashName.String() == "" { - return errors.New(fmt.Sprintf("fetching string: %s and hash: %s names failed\r\nerr1: %v\r\nerr2: %v\r\n ", strName, hashName, err1, err2)) + strName, err := t.Fetcher.FetchString("name", t.Parser.Abi(), contractAddr, -1, nil) + if err != nil || strName == "" { + hashName, err := t.Fetcher.FetchHash("name", t.Parser.Abi(), contractAddr, -1, nil) + if err != nil || hashName.String() == "" { + return errors.New("unable to fetch contract name") // provide CLI prompt here for user to input a contract name? } ctrName = hashName.String() } else { @@ -110,8 +113,8 @@ func (t *eventTransformer) Init() error { info := types.ContractInfo{ Name: ctrName, Address: contractAddr, - Abi: t.Parser.GetAbi(), - ParsedAbi: t.Parser.GetParsedAbi(), + Abi: t.Parser.Abi(), + ParsedAbi: t.Parser.ParsedAbi(), StartingBlock: firstBlock, Events: t.Parser.GetEvents(), Methods: t.Parser.GetMethods(), @@ -129,10 +132,14 @@ func (t *eventTransformer) Init() error { return nil } +// Iterate through contracts, creating a new +// converter for each one and using it to +// convert watched event logs and persist +// them into the postgres db func (tr eventTransformer) Execute() error { for _, contract := range tr.ContractInfo { - c := converter.NewConverter(contract) + tr.Converter.Update(contract) for eventName, filter := range contract.Filters { watchedEvents, err := tr.GetWatchedEvents(eventName) @@ -142,7 +149,7 @@ func (tr eventTransformer) Execute() error { } for _, we := range watchedEvents { - err = c.Convert(*we, contract.Events[eventName]) + err = tr.Converter.Convert(*we, contract.Events[eventName]) if err != nil { return err } diff --git a/pkg/omni/transformer/event_transformer_suite_test.go b/pkg/omni/transformer/event_transformer_suite_test.go new file mode 100644 index 00000000..bfa3b204 --- /dev/null +++ b/pkg/omni/transformer/event_transformer_suite_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformer_test + +import ( + "io/ioutil" + "log" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestTransformer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Transformer Suite Test") +} + +var _ = BeforeSuite(func() { + log.SetOutput(ioutil.Discard) +}) diff --git a/pkg/omni/transformer/event_transformer_test.go b/pkg/omni/transformer/event_transformer_test.go new file mode 100644 index 00000000..00807fd6 --- /dev/null +++ b/pkg/omni/transformer/event_transformer_test.go @@ -0,0 +1,15 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformer_test diff --git a/pkg/omni/transformer/integration_test.go b/pkg/omni/transformer/integration_test.go new file mode 100644 index 00000000..00807fd6 --- /dev/null +++ b/pkg/omni/transformer/integration_test.go @@ -0,0 +1,15 @@ +// Copyright 2018 Vulcanize +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformer_test diff --git a/pkg/omni/types/entities.go b/pkg/omni/types/entities.go index 1a6d87fb..54ab6bca 100644 --- a/pkg/omni/types/entities.go +++ b/pkg/omni/types/entities.go @@ -24,20 +24,19 @@ import ( type Event struct { Name string Anonymous bool - Fields []Field + Fields []*Field Logs map[int64]Log // Map of VulcanizeIdLog to parsed event log } type Method struct { Name string Const bool - Inputs []Field - Outputs []Field + Inputs []*Field + Outputs []*Field } type Field struct { abi.Argument - Value interface{} PgType string } @@ -48,8 +47,9 @@ type Log struct { } func NewEvent(e abi.Event) *Event { - fields := make([]Field, len(e.Inputs)) + fields := make([]*Field, len(e.Inputs)) for i, input := range e.Inputs { + fields[i] = &Field{} fields[i].Name = input.Name fields[i].Type = input.Type fields[i].Indexed = input.Indexed @@ -64,15 +64,17 @@ func NewEvent(e abi.Event) *Event { } func NewMethod(m abi.Method) *Method { - inputs := make([]Field, len(m.Inputs)) + inputs := make([]*Field, len(m.Inputs)) for i, input := range m.Inputs { + inputs[i] = &Field{} inputs[i].Name = input.Name inputs[i].Type = input.Type inputs[i].Indexed = input.Indexed } - outputs := make([]Field, len(m.Outputs)) + outputs := make([]*Field, len(m.Outputs)) for i, output := range m.Outputs { + outputs[i] = &Field{} outputs[i].Name = output.Name outputs[i].Type = output.Type outputs[i].Indexed = output.Indexed diff --git a/vendor/github.com/ethereum/go-ethereum/accounts/abi/argument.go b/vendor/github.com/ethereum/go-ethereum/accounts/abi/argument.go index 4f796fd1..d1333112 100644 --- a/vendor/github.com/ethereum/go-ethereum/accounts/abi/argument.go +++ b/vendor/github.com/ethereum/go-ethereum/accounts/abi/argument.go @@ -192,8 +192,7 @@ func (arguments Arguments) unpackIntoMap(v map[string]interface{}, marshalledVal } for i, arg := range arguments.NonIndexed() { - reflectValue := reflect.ValueOf(marshalledValues[i]) - v[arg.Name] = reflectValue + v[arg.Name] = marshalledValues[i] } return nil