diff --git a/cmd/lightOmniWatcher.go b/cmd/lightOmniWatcher.go index 199eaed2..98f4e6ae 100644 --- a/cmd/lightOmniWatcher.go +++ b/cmd/lightOmniWatcher.go @@ -67,9 +67,10 @@ func lightOmniWatcher() { for _, addr := range contractAddresses { t.SetEvents(addr, contractEvents) t.SetMethods(addr, contractMethods) - t.SetEventAddrs(addr, eventAddrs) - t.SetMethodAddrs(addr, methodAddrs) + t.SetEventArgs(addr, eventArgs) + t.SetMethodArgs(addr, methodArgs) t.SetRange(addr, [2]int64{startingBlockNumber, endingBlockNumber}) + t.SetCreateAddrList(addr, createAddrList) } err := t.Init() @@ -90,11 +91,12 @@ func init() { lightOmniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") lightOmniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address") - lightOmniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched") - lightOmniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled") - lightOmniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses") - lightOmniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses") + lightOmniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched") + lightOmniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled") + lightOmniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified") + lightOmniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here") lightOmniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) lightOmniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists") lightOmniWatcherCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block") + lightOmniWatcherCmd.Flags().BoolVarP(&createAddrList, "create-address-list", "c", false, "Set to true to persist address seen in emitted events into the database") } diff --git a/cmd/omniWatcher.go b/cmd/omniWatcher.go index 566b6672..50f4ccbb 100644 --- a/cmd/omniWatcher.go +++ b/cmd/omniWatcher.go @@ -67,8 +67,8 @@ func omniWatcher() { for _, addr := range contractAddresses { t.SetEvents(addr, contractEvents) t.SetMethods(addr, contractMethods) - t.SetEventAddrs(addr, eventAddrs) - t.SetMethodAddrs(addr, methodAddrs) + t.SetEventArgs(addr, eventArgs) + t.SetMethodArgs(addr, methodArgs) t.SetRange(addr, [2]int64{startingBlockNumber, endingBlockNumber}) } @@ -90,11 +90,12 @@ func init() { omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for") omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address") - omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched") - omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled") - omniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses") - omniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses") + omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched") + omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled") + omniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified") + omniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here") omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`) omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists") omniWatcherCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block") + omniWatcherCmd.Flags().BoolVarP(&createAddrList, "create-address-list", "c", false, "Set to true to persist address seen in emitted events into the database") } diff --git a/cmd/root.go b/cmd/root.go index 43bfff76..f5ae196b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -47,8 +47,9 @@ var ( contractAddresses []string contractEvents []string contractMethods []string - eventAddrs []string - methodAddrs []string + eventArgs []string + methodArgs []string + createAddrList bool ) var rootCmd = &cobra.Command{ diff --git a/documentation/ens_questions.md b/documentation/ens_questions.md index 036d78c8..d2d8660e 100644 --- a/documentation/ens_questions.md +++ b/documentation/ens_questions.md @@ -1,41 +1,46 @@ +Run lightSync, starting at the blockheight the ENS registry was published at. +`./vulcanize lightSync --config=./environments/ --starting-block-number=3327417` + + 1. What is the label hash of this domain? - Q. Does this mean for a given namehash of "a.b.c" find keccak256(a), keccak256(b), and keccak256(c)? Do we know the parent domain and/or owner address? - 1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract and filter for the root domain and/or owner address to narrow search - `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner` - 2. For each node + label pair we find emitted, calculate the keccak256(abi.encodePacked(node, label)) and see if it matches our namehash - 3. If it does, hash(label) is our answer + Q. Does this mean for a given namehash of "a.b.c" where we don't know what "c" is find keccak256(c)? Do we know the parent domain and/or owner address? + 1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract and filter for the parent node and/or owner address to narrow search + `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner` + 2. For each node + label pair we find emitted, calculate the keccak256(abi.encodePacked(node, label)) and see if it matches our domain's namehash + 3. If it does, keccak256(label) is our label hash 2. What is the parent domain of this domain? 1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract - `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner` - 2. Filter for our label (domain) and collect the node (parent domain namehash) that was emitted with it - 3. Call the Registry's resolver(bytes32 node) method for the parent node to find the parent domain's Resolver - 4. Call its Resolver's name(bytes32 node) method for the parent node to find the parent domain's name + `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner` + 2. If we know our label, filter for it and collect the node (parent domain namehash) that was emitted with it + 3. If we don't know our label I believe we need to take a similar aqpproach as in section 1 where we try every keccak256(abi.encodePacked(node, label)) until we find the namehash for our domain + 4. Call the Registry's resolver(bytes32 node) method for the parent node to find the parent domain's Resolver + 5. Call its Resolver's name(bytes32 node) method for the parent node to find the parent domain's name 3. What are the subdomains of this domain? - 1. Watch NewOwner events of the ENS Registry contract - `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner` + 1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract + `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner` 2. Filter for our node (domain) and collect all the labels emitted with it - 3. Calculate subdomain hashes: subnode = keccak256(abi.encodePacked(node, label)); + 3. Calculate subdomain hashes: subnode = keccak256(abi.encodePacked(node, label)) 4. Call the Registry's resolver(bytes32 node) method for a subnode to find the subdomain's Resolver 5. Call its Resolver's name(bytes32 node) method for a subnode to find the subdomain's name 4. What domains does this address own? 1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) and Transfer(bytes32 indexed node, address owner) events of the ENS Registry contract and filter for the address - `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-eevents=NewOwner --contract-events=Transfer --event-filter-addresses=
` - 2. Generate list of all nodes this address has ever owned - 3. Check which of these they still own at a given blockheight by iterating over the list and calling the owner(bytes32 node) method + `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-eevents=NewOwner --contract-events=Transfer --event-filter-addresses=
` + 2. Collect node and label values; calculate subnodes = keccak256(abi.encodePacked(node, label)) + 3. Aggregate nodes and subnodes into a list and check which of these they still own at a given blockheight by iterating over the list and calling the Registry's owner(bytes32 node) method 4. Call the Registry's resolver(bytes32 node) method for a node to find the domain's Resolver 5. Call its Resolver's name(bytes32 node) method for the node to find the domain's name 5. What names point to this address? Q. Is this in terms of which ENS nodes point to a given Resolver address? E.g. All nodes where the ENS records[node].resolver == address? Or is this in terms of Resolver records? E.g. All the records[node].names where the Resolver records[node].addr == address 1. In the former case, watch NewResolver(bytes32 indexed node, address resolver) events of the Registry and filter for the account address - `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x1da022710dF5002339274AaDEe8D58218e9D6AB5 --contract-events=NewResolver --event-filter-addresses=
` + `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewResolver --event-filter-addresses=
` 2. Generate a list of nodes that have pointed to this resolver address - 3. Check which of these names still point at the address by iterating over the list and calling the resolver(bytes32 node) method + 3. Check which of these still point at the address by iterating over the list and calling the Registry's resolver(bytes32 node) method 1. In the latter case, watch AddrChanged(bytes32 indexed node, address a) events of the Resolver and filter for the account address - `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x1da022710dF5002339274AaDEe8D58218e9D6AB5 --contract-events=AddrChanged --event-filter-addresses=
` + `./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3648359 --contract-address=0x1da022710dF5002339274AaDEe8D58218e9D6AB5 --contract-events=AddrChanged --event-filter-addresses=
` 2. Generate our list of nodes that have pointed towards our address 3. Check which of these they still own at a given blockheight by iterating over the list and calling the Resolver's addr(bytes32 node) method 4. We can then fetch the string names of these nodes using the Resolver's name(bytes32 node) method. @@ -48,9 +53,9 @@ we could also perform []byte filtering on the events and automate polling of eve currently working on adding this in, and once it is in you would be able to automate more of the steps in these processes. E.g. you will be able to run -`./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --contract-events=Transfer --event-args=
--contract-methods=owner` +`./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --contract-events=Transfer --event-args=
--contract-methods=owner` To automate the process in question 4 through step 3 (it will collect node []byte values emitted from the events it watches and then use those to call the owner method, persisting the results) Or -`./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --event-args=` +`./vulcanize lightOmniWacther --config=./environments/ --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --event-args=` To provide automated filtering for node []byte values in question 3. \ No newline at end of file diff --git a/pkg/omni/full/converter/converter.go b/pkg/omni/full/converter/converter.go index 547acbff..e4bc0d6a 100644 --- a/pkg/omni/full/converter/converter.go +++ b/pkg/omni/full/converter/converter.go @@ -69,7 +69,9 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( } strValues := make(map[string]string, len(values)) - + seenBytes := make([]interface{}, 0, len(values)) + seenAddrs := make([]interface{}, 0, len(values)) + seenHashes := make([]interface{}, 0, len(values)) for fieldName, input := range values { // Postgres cannot handle custom types, resolve to strings switch input.(type) { @@ -79,10 +81,11 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( case common.Address: a := input.(common.Address) strValues[fieldName] = a.String() - c.ContractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses + seenAddrs = append(seenAddrs, a) case common.Hash: h := input.(common.Hash) strValues[fieldName] = h.String() + seenHashes = append(seenHashes, h) case string: strValues[fieldName] = input.(string) case bool: @@ -90,6 +93,7 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( case []byte: b := input.([]byte) strValues[fieldName] = string(b) + seenBytes = append(seenBytes, b) case byte: b := input.(byte) strValues[fieldName] = string(b) @@ -107,6 +111,17 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) ( Tx: watchedEvent.TxHash, } + // Cache emitted values if their caching is turned on + if c.ContractInfo.EmittedAddrs != nil { + c.ContractInfo.AddEmittedAddr(seenAddrs...) + } + if c.ContractInfo.EmittedHashes != nil { + c.ContractInfo.AddEmittedHash(seenHashes...) + } + if c.ContractInfo.EmittedBytes != nil { + c.ContractInfo.AddEmittedBytes(seenBytes...) + } + return eventLog, nil } diff --git a/pkg/omni/full/converter/converter_test.go b/pkg/omni/full/converter/converter_test.go index c5bfc3e7..68e2e278 100644 --- a/pkg/omni/full/converter/converter_test.go +++ b/pkg/omni/full/converter/converter_test.go @@ -81,21 +81,21 @@ var _ = Describe("Converter", func() { _, err := c.Convert(mocks.MockTranferEvent, event) Expect(err).ToNot(HaveOccurred()) - b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"] + b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"] + b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - _, ok = con.TknHolderAddrs["0x"] + _, ok = con.EmittedAddrs[common.HexToAddress("0x")] Expect(ok).To(Equal(false)) - _, ok = con.TknHolderAddrs[""] + _, ok = con.EmittedAddrs[""] Expect(ok).To(Equal(false)) - _, ok = con.TknHolderAddrs["0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP"] + _, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] Expect(ok).To(Equal(false)) }) diff --git a/pkg/omni/full/transformer/transformer.go b/pkg/omni/full/transformer/transformer.go index 2191954f..d18c6482 100644 --- a/pkg/omni/full/transformer/transformer.go +++ b/pkg/omni/full/transformer/transformer.go @@ -18,7 +18,7 @@ package transformer import ( "errors" - "fmt" + "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -63,8 +63,11 @@ type transformer struct { // Lists of addresses to filter event or method data // before persisting; if empty no filter is applied - EventAddrs map[string][]string - MethodAddrs map[string][]string + EventArgs map[string][]string + MethodArgs map[string][]string + + // Whether or not to create a list of token holder addresses for the contract in postgres + CreateAddrList map[string]bool } // Transformer takes in config for blockchain, database, and network id @@ -81,8 +84,8 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf WatchedEvents: map[string][]string{}, WantedMethods: map[string][]string{}, ContractRanges: map[string][2]int64{}, - EventAddrs: map[string][]string{}, - MethodAddrs: map[string][]string{}, + EventArgs: map[string][]string{}, + MethodArgs: map[string][]string{}, } } @@ -116,25 +119,22 @@ func (t *transformer) Init() error { lastBlock = t.ContractRanges[contractAddr][1] } - // Get contract name + // Get contract name if it has one var name = new(string) - err = t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock) - if err != nil { - return errors.New(fmt.Sprintf("unable to fetch contract name: %v\r\n", err)) - } + t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock) - // Remove any accidental duplicate inputs in filter addresses - EventAddrs := map[string]bool{} - for _, addr := range t.EventAddrs[contractAddr] { - EventAddrs[addr] = true + // Remove any potential accidental duplicate inputs in arg filter values + eventArgs := map[string]bool{} + for _, arg := range t.EventArgs[contractAddr] { + eventArgs[arg] = true } - MethodAddrs := map[string]bool{} - for _, addr := range t.MethodAddrs[contractAddr] { - MethodAddrs[addr] = true + methodArgs := map[string]bool{} + for _, arg := range t.MethodArgs[contractAddr] { + methodArgs[arg] = true } // Aggregate info into contract object - info := &contract.Contract{ + info := contract.Contract{ Name: *name, Network: t.Network, Address: contractAddr, @@ -143,11 +143,11 @@ func (t *transformer) Init() error { StartingBlock: firstBlock, LastBlock: lastBlock, Events: t.GetEvents(subset), - Methods: t.GetAddrMethods(t.WantedMethods[contractAddr]), - EventAddrs: EventAddrs, - MethodAddrs: MethodAddrs, - TknHolderAddrs: map[string]bool{}, - } + Methods: t.GetSelectMethods(t.WantedMethods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + CreateAddrList: t.CreateAddrList[contractAddr], + }.Init() // Use info to create filters err = info.GenerateFilters() @@ -222,26 +222,31 @@ func (tr transformer) Execute() error { } // Used to set which contract addresses and which of their events to watch -func (t *transformer) SetEvents(contractAddr string, filterSet []string) { - t.WatchedEvents[contractAddr] = filterSet +func (tr *transformer) SetEvents(contractAddr string, filterSet []string) { + tr.WatchedEvents[contractAddr] = filterSet } // Used to set subset of account addresses to watch events for -func (t *transformer) SetEventAddrs(contractAddr string, filterSet []string) { - t.EventAddrs[contractAddr] = filterSet +func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) { + tr.EventArgs[contractAddr] = filterSet } // Used to set which contract addresses and which of their methods to call -func (t *transformer) SetMethods(contractAddr string, filterSet []string) { - t.WantedMethods[contractAddr] = filterSet +func (tr *transformer) SetMethods(contractAddr string, filterSet []string) { + tr.WantedMethods[contractAddr] = filterSet } // Used to set subset of account addresses to poll methods on -func (t *transformer) SetMethodAddrs(contractAddr string, filterSet []string) { - t.MethodAddrs[contractAddr] = filterSet +func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) { + tr.MethodArgs[contractAddr] = filterSet } // Used to set the block range to watch for a given address -func (t *transformer) SetRange(contractAddr string, rng [2]int64) { - t.ContractRanges[contractAddr] = rng +func (tr *transformer) SetRange(contractAddr string, rng [2]int64) { + tr.ContractRanges[contractAddr] = rng +} + +// Used to set the block range to watch for a given address +func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) { + tr.CreateAddrList[contractAddr] = on } diff --git a/pkg/omni/full/transformer/transformer_test.go b/pkg/omni/full/transformer/transformer_test.go index ef1f9327..ac989c9e 100644 --- a/pkg/omni/full/transformer/transformer_test.go +++ b/pkg/omni/full/transformer/transformer_test.go @@ -21,6 +21,7 @@ import ( "math/rand" "time" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -62,8 +63,8 @@ var _ = Describe("Transformer", func() { It("Sets which account addresses to watch events for", func() { eventAddrs := []string{"test1", "test2"} t := transformer.NewTransformer("", blockChain, db) - t.SetEventAddrs(constants.TusdContractAddress, eventAddrs) - Expect(t.EventAddrs[constants.TusdContractAddress]).To(Equal(eventAddrs)) + t.SetEventArgs(constants.TusdContractAddress, eventAddrs) + Expect(t.EventArgs[constants.TusdContractAddress]).To(Equal(eventAddrs)) }) }) @@ -80,8 +81,8 @@ var _ = Describe("Transformer", func() { It("Sets which account addresses to poll methods against", func() { methodAddrs := []string{"test1", "test2"} t := transformer.NewTransformer("", blockChain, db) - t.SetMethodAddrs(constants.TusdContractAddress, methodAddrs) - Expect(t.MethodAddrs[constants.TusdContractAddress]).To(Equal(methodAddrs)) + t.SetMethodArgs(constants.TusdContractAddress, methodAddrs) + Expect(t.MethodArgs[constants.TusdContractAddress]).To(Equal(methodAddrs)) }) }) @@ -159,10 +160,10 @@ var _ = Describe("Transformer", func() { Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146")) }) - It("Keeps track of contract-related addresses while transforming event data", func() { + It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { t := transformer.NewTransformer("", blockChain, db) t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -172,18 +173,24 @@ var _ = Describe("Transformer", func() { err = t.Execute() Expect(err).ToNot(HaveOccurred()) - b, ok := c.TknHolderAddrs["0x000000000000000000000000000000000000Af21"] + b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"] + b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - _, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843b1234567890"] + _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] Expect(ok).To(Equal(false)) - _, ok = c.TknHolderAddrs["0x"] + _, ok = c.EmittedAddrs[common.HexToAddress("0x")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[""] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] Expect(ok).To(Equal(false)) }) @@ -199,7 +206,12 @@ var _ = Describe("Transformer", func() { res := test_helpers.BalanceOf{} - err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", constants.TusdContractAddress)).StructScan(&res) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", constants.TusdContractAddress)).StructScan(&res) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Balance).To(Equal("0")) + Expect(res.TokenName).To(Equal("TrueUSD")) + + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", constants.TusdContractAddress)).StructScan(&res) Expect(err).ToNot(HaveOccurred()) Expect(res.Balance).To(Equal("0")) Expect(res.TokenName).To(Equal("TrueUSD")) diff --git a/pkg/omni/light/converter/converter.go b/pkg/omni/light/converter/converter.go index a92ee665..9fd5b7c7 100644 --- a/pkg/omni/light/converter/converter.go +++ b/pkg/omni/light/converter/converter.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" gethTypes "github.com/ethereum/go-ethereum/core/types" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" @@ -67,6 +68,9 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in } strValues := make(map[string]string, len(values)) + seenBytes := make([]interface{}, 0, len(values)) + seenAddrs := make([]interface{}, 0, len(values)) + seenHashes := make([]interface{}, 0, len(values)) for fieldName, input := range values { // Postgres cannot handle custom types, resolve everything to strings switch input.(type) { @@ -76,17 +80,19 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in case common.Address: a := input.(common.Address) strValues[fieldName] = a.String() - c.ContractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses + seenAddrs = append(seenAddrs, a) case common.Hash: h := input.(common.Hash) strValues[fieldName] = h.String() + seenHashes = append(seenHashes, h) case string: strValues[fieldName] = input.(string) case bool: strValues[fieldName] = strconv.FormatBool(input.(bool)) case []byte: b := input.([]byte) - strValues[fieldName] = string(b) + strValues[fieldName] = hexutil.Encode(b) + seenBytes = append(seenBytes, b) case byte: b := input.(byte) strValues[fieldName] = string(b) @@ -109,6 +115,17 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in TransactionIndex: log.TxIndex, Id: headerID, }) + + // Cache emitted values if their caching is turned on + if c.ContractInfo.EmittedAddrs != nil { + c.ContractInfo.AddEmittedAddr(seenAddrs...) + } + if c.ContractInfo.EmittedHashes != nil { + c.ContractInfo.AddEmittedHash(seenHashes...) + } + if c.ContractInfo.EmittedBytes != nil { + c.ContractInfo.AddEmittedBytes(seenBytes...) + } } } diff --git a/pkg/omni/light/converter/converter_test.go b/pkg/omni/light/converter/converter_test.go index 96a375ca..3ee988cd 100644 --- a/pkg/omni/light/converter/converter_test.go +++ b/pkg/omni/light/converter/converter_test.go @@ -84,21 +84,21 @@ var _ = Describe("Converter", func() { _, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232) Expect(err).ToNot(HaveOccurred()) - b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"] + b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"] + b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - _, ok = con.TknHolderAddrs["0x"] + _, ok = con.EmittedAddrs[common.HexToAddress("0x")] Expect(ok).To(Equal(false)) - _, ok = con.TknHolderAddrs[""] + _, ok = con.EmittedAddrs[""] Expect(ok).To(Equal(false)) - _, ok = con.TknHolderAddrs["0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP"] + _, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] Expect(ok).To(Equal(false)) }) diff --git a/pkg/omni/light/repository/header_repository.go b/pkg/omni/light/repository/header_repository.go index 8e9ce00f..8818003c 100644 --- a/pkg/omni/light/repository/header_repository.go +++ b/pkg/omni/light/repository/header_repository.go @@ -18,6 +18,7 @@ package repository import ( "database/sql" + "github.com/hashicorp/golang-lru" "github.com/vulcanize/vulcanizedb/pkg/core" @@ -85,7 +86,8 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock LEFT JOIN checked_headers on headers.id = header_id WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE) AND headers.block_number >= $1 - AND headers.eth_node_fingerprint = $2` + AND headers.eth_node_fingerprint = $2 + ORDER BY headers.block_number` err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID) } else { query = `SELECT headers.id, headers.block_number, headers.hash FROM headers @@ -93,7 +95,8 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE) AND headers.block_number >= $1 AND headers.block_number <= $2 - AND headers.eth_node_fingerprint = $3` + AND headers.eth_node_fingerprint = $3 + ORDER BY headers.block_number` err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID) } diff --git a/pkg/omni/light/retriever/block_retriever.go b/pkg/omni/light/retriever/block_retriever.go index 3675b8c6..a871278e 100644 --- a/pkg/omni/light/retriever/block_retriever.go +++ b/pkg/omni/light/retriever/block_retriever.go @@ -42,7 +42,7 @@ func (r *blockRetriever) RetrieveFirstBlock() (int64, error) { var firstBlock int err := r.db.Get( &firstBlock, - "SELECT block_number FROM headers ORDER BY block_number ASC LIMIT 1", + "SELECT block_number FROM headers ORDER BY block_number LIMIT 1", ) return int64(firstBlock), err diff --git a/pkg/omni/light/transformer/transformer.go b/pkg/omni/light/transformer/transformer.go index 26804ea7..161a41c7 100644 --- a/pkg/omni/light/transformer/transformer.go +++ b/pkg/omni/light/transformer/transformer.go @@ -18,7 +18,6 @@ package transformer import ( "errors" - "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" "strings" "github.com/ethereum/go-ethereum/common" @@ -30,6 +29,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" "github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller" srep "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository" @@ -67,8 +67,11 @@ type transformer struct { // Lists of addresses to filter event or method data // before persisting; if empty no filter is applied - EventAddrs map[string][]string - MethodAddrs map[string][]string + EventArgs map[string][]string + MethodArgs map[string][]string + + // Whether or not to create a list of token holder addresses for the contract in postgres + CreateAddrList map[string]bool } // Transformer takes in config for blockchain, database, and network id @@ -86,8 +89,9 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf WatchedEvents: map[string][]string{}, WantedMethods: map[string][]string{}, ContractRanges: map[string][2]int64{}, - EventAddrs: map[string][]string{}, - MethodAddrs: map[string][]string{}, + EventArgs: map[string][]string{}, + MethodArgs: map[string][]string{}, + CreateAddrList: map[string]bool{}, } } @@ -126,18 +130,18 @@ func (tr *transformer) Init() error { var name = new(string) tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock) - // Remove any potential accidental duplicate inputs in filter addresses - EventAddrs := map[string]bool{} - for _, addr := range tr.EventAddrs[contractAddr] { - EventAddrs[addr] = true + // Remove any potential accidental duplicate inputs in arg filter values + eventArgs := map[string]bool{} + for _, arg := range tr.EventArgs[contractAddr] { + eventArgs[arg] = true } - MethodAddrs := map[string]bool{} - for _, addr := range tr.MethodAddrs[contractAddr] { - MethodAddrs[addr] = true + methodArgs := map[string]bool{} + for _, arg := range tr.MethodArgs[contractAddr] { + methodArgs[arg] = true } - // Aggregate info into contract object - info := &contract.Contract{ + // Aggregate info into contract object and store for execution + tr.Contracts[contractAddr] = contract.Contract{ Name: *name, Network: tr.Network, Address: contractAddr, @@ -146,14 +150,11 @@ func (tr *transformer) Init() error { StartingBlock: firstBlock, LastBlock: lastBlock, Events: tr.GetEvents(subset), - Methods: tr.GetAddrMethods(tr.WantedMethods[contractAddr]), - EventAddrs: EventAddrs, - MethodAddrs: MethodAddrs, - TknHolderAddrs: map[string]bool{}, - } - - // Store contract info for execution - tr.Contracts[contractAddr] = info + Methods: tr.GetSelectMethods(tr.WantedMethods[contractAddr]), + FilterArgs: eventArgs, + MethodArgs: methodArgs, + CreateAddrList: tr.CreateAddrList[contractAddr], + }.Init() } return nil @@ -216,14 +217,14 @@ func (tr *transformer) Execute() error { if err != nil { return err } + + // Poll contract methods at this header's block height + // with arguments collected from event logs up to this point + if err := tr.PollContractAt(*con, header.BlockNumber); err != nil { + return err + } } } - // After persisting all watched event logs - // poller polls select contract methods - // and persists the results into custom pg tables - if err := tr.PollContract(*con); err != nil { - return err - } } return nil @@ -235,8 +236,8 @@ func (tr *transformer) SetEvents(contractAddr string, filterSet []string) { } // Used to set subset of account addresses to watch events for -func (tr *transformer) SetEventAddrs(contractAddr string, filterSet []string) { - tr.EventAddrs[contractAddr] = filterSet +func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) { + tr.EventArgs[contractAddr] = filterSet } // Used to set which contract addresses and which of their methods to call @@ -245,11 +246,16 @@ func (tr *transformer) SetMethods(contractAddr string, filterSet []string) { } // Used to set subset of account addresses to poll methods on -func (tr *transformer) SetMethodAddrs(contractAddr string, filterSet []string) { - tr.MethodAddrs[contractAddr] = filterSet +func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) { + tr.MethodArgs[contractAddr] = filterSet } // Used to set the block range to watch for a given address func (tr *transformer) SetRange(contractAddr string, rng [2]int64) { tr.ContractRanges[contractAddr] = rng } + +// Used to set the block range to watch for a given address +func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) { + tr.CreateAddrList[contractAddr] = on +} diff --git a/pkg/omni/light/transformer/transformer_test.go b/pkg/omni/light/transformer/transformer_test.go index 31a6574e..7df83858 100644 --- a/pkg/omni/light/transformer/transformer_test.go +++ b/pkg/omni/light/transformer/transformer_test.go @@ -18,6 +18,8 @@ package transformer_test import ( "fmt" + + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -59,8 +61,8 @@ var _ = Describe("Transformer", func() { It("Sets which account addresses to watch events for", func() { eventAddrs := []string{"test1", "test2"} t := transformer.NewTransformer("", blockChain, db) - t.SetEventAddrs(constants.TusdContractAddress, eventAddrs) - Expect(t.EventAddrs[constants.TusdContractAddress]).To(Equal(eventAddrs)) + t.SetEventArgs(constants.TusdContractAddress, eventAddrs) + Expect(t.EventArgs[constants.TusdContractAddress]).To(Equal(eventAddrs)) }) }) @@ -77,8 +79,8 @@ var _ = Describe("Transformer", func() { It("Sets which account addresses to poll methods against", func() { methodAddrs := []string{"test1", "test2"} t := transformer.NewTransformer("", blockChain, db) - t.SetMethodAddrs(constants.TusdContractAddress, methodAddrs) - Expect(t.MethodAddrs[constants.TusdContractAddress]).To(Equal(methodAddrs)) + t.SetMethodArgs(constants.TusdContractAddress, methodAddrs) + Expect(t.MethodArgs[constants.TusdContractAddress]).To(Equal(methodAddrs)) }) }) @@ -164,10 +166,10 @@ var _ = Describe("Transformer", func() { Expect(log.Value).To(Equal("9998940000000000000000")) }) - It("Keeps track of contract-related addresses while transforming event data", func() { + It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() { t := transformer.NewTransformer("", blockChain, db) t.SetEvents(constants.TusdContractAddress, []string{"Transfer"}) - t.SetMethods(constants.TusdContractAddress, nil) + t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"}) err = t.Init() Expect(err).ToNot(HaveOccurred()) @@ -177,18 +179,24 @@ var _ = Describe("Transformer", func() { err = t.Execute() Expect(err).ToNot(HaveOccurred()) - b, ok := c.TknHolderAddrs["0x1062a747393198f70F71ec65A582423Dba7E5Ab3"] + b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = c.TknHolderAddrs["0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"] + b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - _, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843b1234567890"] + _, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")] Expect(ok).To(Equal(false)) - _, ok = c.TknHolderAddrs["0x"] + _, ok = c.EmittedAddrs[common.HexToAddress("0x")] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[""] + Expect(ok).To(Equal(false)) + + _, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")] Expect(ok).To(Equal(false)) }) @@ -204,6 +212,17 @@ var _ = Describe("Transformer", func() { res := test_helpers.BalanceOf{} + c, ok := t.Contracts[constants.TusdContractAddress] + Expect(ok).To(Equal(true)) + + b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + + b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")] + Expect(ok).To(Equal(true)) + Expect(b).To(Equal(true)) + err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", constants.TusdContractAddress)).StructScan(&res) Expect(err).ToNot(HaveOccurred()) Expect(res.Balance).To(Equal("55849938025000000000000")) diff --git a/pkg/omni/shared/constants/constants.go b/pkg/omni/shared/constants/constants.go index 9be22830..553cd1ce 100644 --- a/pkg/omni/shared/constants/constants.go +++ b/pkg/omni/shared/constants/constants.go @@ -17,6 +17,7 @@ package constants import ( + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers" @@ -65,6 +66,7 @@ func (e Event) Signature() string { // Contract Addresses var DaiContractAddress = "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E" +var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b" // Contract Owner var DaiContractOwner = "0x0000000000000000000000000000000000000000" @@ -75,6 +77,15 @@ var DaiAbiString = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"nam var TusdAbiString = `[{"constant":true,"inputs":[],"name":"burnMin","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"spender","type":"address"},{"name":"value","type":"uint256"}],"name":"approve","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"owner","type":"address"},{"name":"spender","type":"address"}],"name":"delegateAllowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"burnFeeFlat","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_canReceiveMintWhiteList","type":"address"},{"name":"_canBurnWhiteList","type":"address"},{"name":"_blackList","type":"address"},{"name":"_noFeesList","type":"address"}],"name":"setLists","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"token","type":"address"}],"name":"reclaimToken","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"newContract","type":"address"}],"name":"delegateToNewContract","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_transferFeeNumerator","type":"uint80"},{"name":"_transferFeeDenominator","type":"uint80"},{"name":"_mintFeeNumerator","type":"uint80"},{"name":"_mintFeeDenominator","type":"uint80"},{"name":"_mintFeeFlat","type":"uint256"},{"name":"_burnFeeNumerator","type":"uint80"},{"name":"_burnFeeDenominator","type":"uint80"},{"name":"_burnFeeFlat","type":"uint256"}],"name":"changeStakingFees","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"canReceiveMintWhiteList","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"from","type":"address"},{"name":"to","type":"address"},{"name":"value","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"delegatedFrom","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"spender","type":"address"},{"name":"value","type":"uint256"},{"name":"origSender","type":"address"}],"name":"delegateApprove","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"contractAddr","type":"address"}],"name":"reclaimContract","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint8"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"allowances","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"unpause","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"mint","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_value","type":"uint256"}],"name":"burn","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"who","type":"address"}],"name":"delegateBalanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"from","type":"address"},{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"origSender","type":"address"}],"name":"delegateTransferFrom","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"claimOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"sheet","type":"address"}],"name":"setBalanceSheet","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"spender","type":"address"},{"name":"addedValue","type":"uint256"},{"name":"origSender","type":"address"}],"name":"delegateIncreaseApproval","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"burnFeeNumerator","outputs":[{"name":"","type":"uint80"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"canBurnWhiteList","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"burnMax","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"paused","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mintFeeDenominator","outputs":[{"name":"","type":"uint80"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"staker","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"}],"name":"setDelegatedFrom","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"spender","type":"address"},{"name":"subtractedValue","type":"uint256"}],"name":"decreaseApproval","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"noFeesList","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"who","type":"address"}],"name":"balanceOf","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"newMin","type":"uint256"},{"name":"newMax","type":"uint256"}],"name":"changeBurnBounds","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"delegateTotalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"balances","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"pause","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_name","type":"string"},{"name":"_symbol","type":"string"}],"name":"changeName","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"mintFeeNumerator","outputs":[{"name":"","type":"uint80"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"transferFeeNumerator","outputs":[{"name":"","type":"uint80"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"spender","type":"address"},{"name":"subtractedValue","type":"uint256"},{"name":"origSender","type":"address"}],"name":"delegateDecreaseApproval","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"origSender","type":"address"}],"name":"delegateTransfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"reclaimEther","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"}],"name":"transfer","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"newStaker","type":"address"}],"name":"changeStaker","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"account","type":"address"}],"name":"wipeBlacklistedAccount","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"from_","type":"address"},{"name":"value_","type":"uint256"},{"name":"data_","type":"bytes"}],"name":"tokenFallback","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"burnFeeDenominator","outputs":[{"name":"","type":"uint80"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"delegate","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"blackList","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"transferFeeDenominator","outputs":[{"name":"","type":"uint80"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"mintFeeFlat","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"spender","type":"address"},{"name":"addedValue","type":"uint256"}],"name":"increaseApproval","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"spender","type":"address"}],"name":"allowance","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"pendingOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"sheet","type":"address"}],"name":"setAllowanceSheet","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"payable":false,"stateMutability":"nonpayable","type":"fallback"},{"anonymous":false,"inputs":[{"indexed":false,"name":"newMin","type":"uint256"},{"indexed":false,"name":"newMax","type":"uint256"}],"name":"ChangeBurnBoundsEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"amount","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"account","type":"address"},{"indexed":false,"name":"balance","type":"uint256"}],"name":"WipedAccount","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"newContract","type":"address"}],"name":"DelegatedTo","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"burner","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[],"name":"Pause","type":"event"},{"anonymous":false,"inputs":[],"name":"Unpause","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"previousOwner","type":"address"},{"indexed":true,"name":"newOwner","type":"address"}],"name":"OwnershipTransferred","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"owner","type":"address"},{"indexed":true,"name":"spender","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"from","type":"address"},{"indexed":true,"name":"to","type":"address"},{"indexed":false,"name":"value","type":"uint256"}],"name":"Transfer","type":"event"}]` +var ENSAbiString = `[{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"resolver","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"label","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setSubnodeOwner","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"ttl","type":"uint64"}],"name":"setTTL","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[{"name":"node","type":"bytes32"}],"name":"ttl","outputs":[{"name":"","type":"uint64"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"resolver","type":"address"}],"name":"setResolver","outputs":[],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"node","type":"bytes32"},{"name":"owner","type":"address"}],"name":"setOwner","outputs":[],"payable":false,"type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"Transfer","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":true,"name":"label","type":"bytes32"},{"indexed":false,"name":"owner","type":"address"}],"name":"NewOwner","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"resolver","type":"address"}],"name":"NewResolver","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"ttl","type":"uint64"}],"name":"NewTTL","type":"event"}]` + +// Look-up table for ABI strings +var Abis = map[common.Address]string{ + common.HexToAddress("0x314159265dD8dbb310642f98f50C066173C1259b"): ENSAbiString, + common.HexToAddress("0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"): TusdAbiString, + common.HexToAddress("0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359"): DaiAbiString, +} + // Filters // To add additional filter parameters, filter by other Topics e.g. for a Transfer event filter Topics[1] to filter for a specific 'from' address var DaiERC20Filters = []filters.LogFilter{ diff --git a/pkg/omni/shared/contract/contract.go b/pkg/omni/shared/contract/contract.go index d6812048..a7f66936 100644 --- a/pkg/omni/shared/contract/contract.go +++ b/pkg/omni/shared/contract/contract.go @@ -20,6 +20,8 @@ import ( "errors" "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/filters" @@ -38,13 +40,42 @@ type Contract struct { ParsedAbi abi.ABI // Parsed abi Events map[string]types.Event // Map of events to their names Methods map[string]types.Method // Map of methods to their names - Filters map[string]filters.LogFilter // Map of event filters to their names - EventAddrs map[string]bool // User-input list of account addresses to watch events for - MethodAddrs map[string]bool // User-input list of account addresses to poll methods for - TknHolderAddrs map[string]bool // List of all contract-associated addresses, populated as events are transformed + Filters map[string]filters.LogFilter // Map of event filters to their names; used only for full sync watcher + FilterArgs map[string]bool // User-input list of values to filter event logs for + MethodArgs map[string]bool // User-input list of values to limit method polling to + EmittedAddrs map[interface{}]bool // List of all unique addresses collected from converted event logs + EmittedBytes map[interface{}]bool // List of all unique bytes collected from converted event logs + EmittedHashes map[interface{}]bool // List of all unique hashes collected from converted event logs + CreateAddrList bool // Whether or not to persist address list to postgres } -// Use contract info to generate event filters +// If we will be calling methods that use addr, hash, or byte arrays +// as arguments then we initialize map to hold these types of values +func (c Contract) Init() *Contract { + for _, method := range c.Methods { + for _, arg := range method.Args { + switch arg.Type.T { + case abi.AddressTy: + c.EmittedAddrs = map[interface{}]bool{} + case abi.HashTy: + c.EmittedHashes = map[interface{}]bool{} + case abi.BytesTy, abi.FixedBytesTy: + c.EmittedBytes = map[interface{}]bool{} + default: + } + } + } + + // If we are creating an address list in postgres + // we initialize the map despite what method call, if any + if c.CreateAddrList { + c.EmittedAddrs = map[interface{}]bool{} + } + + return &c +} + +// Use contract info to generate event filters - full sync omni watcher only func (c *Contract) GenerateFilters() error { c.Filters = map[string]filters.LogFilter{} @@ -65,39 +96,45 @@ func (c *Contract) GenerateFilters() error { return nil } -// Returns true if address is in list of addresses to +// Returns true if address is in list of arguments to // filter events for or if no filtering is specified -func (c *Contract) IsEventAddr(addr string) bool { - if c.EventAddrs == nil { +func (c *Contract) WantedEventArg(arg string) bool { + if c.FilterArgs == nil { return false - } else if len(c.EventAddrs) == 0 { + } else if len(c.FilterArgs) == 0 { return true - } else if a, ok := c.EventAddrs[addr]; ok { + } else if a, ok := c.FilterArgs[arg]; ok { return a } return false } -// Returns true if address is in list of addresses to -// poll methods for or if no filtering is specified -func (c *Contract) IsMethodAddr(addr string) bool { - if c.MethodAddrs == nil { +// Returns true if address is in list of arguments to +// poll methods with or if no filtering is specified +func (c *Contract) WantedMethodArg(arg interface{}) bool { + if c.MethodArgs == nil { return false - } else if len(c.MethodAddrs) == 0 { + } else if len(c.MethodArgs) == 0 { return true - } else if a, ok := c.MethodAddrs[addr]; ok { + } + + // resolve interface to one of the three types we handle as arguments + str := StringifyArg(arg) + + // See if it's hex string has been filtered for + if a, ok := c.MethodArgs[str]; ok { return a } return false } -// Returns true if mapping value matches filtered for address or if not filter exists +// Returns true if any mapping value matches filtered for address or if no filter exists // Used to check if an event log name-value mapping should be filtered or not func (c *Contract) PassesEventFilter(args map[string]string) bool { for _, arg := range args { - if c.IsEventAddr(arg) { + if c.WantedEventArg(arg) { return true } } @@ -105,10 +142,47 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool { return false } -// Used to add an address to the token holder address list -// if it is on the method polling list or the filter is open -func (c *Contract) AddTokenHolderAddress(addr string) { - if c.TknHolderAddrs != nil && c.IsMethodAddr(addr) { - c.TknHolderAddrs[addr] = true +// Add event emitted address to our list if it passes filter and method polling is on +func (c *Contract) AddEmittedAddr(addresses ...interface{}) { + for _, addr := range addresses { + if c.WantedMethodArg(addr) && c.Methods != nil { + c.EmittedAddrs[addr] = true + } } } + +// Add event emitted hash to our list if it passes filter and method polling is on +func (c *Contract) AddEmittedHash(hashes ...interface{}) { + for _, hash := range hashes { + if c.WantedMethodArg(hash) && c.Methods != nil { + c.EmittedHashes[hash] = true + } + } +} + +// Add event emitted bytes to our list if it passes filter and method polling is on +func (c *Contract) AddEmittedBytes(byteArrays ...interface{}) { + for _, bytes := range byteArrays { + if c.WantedMethodArg(bytes) && c.Methods != nil { + c.EmittedBytes[bytes] = true + } + } +} + +func StringifyArg(arg interface{}) (str string) { + switch arg.(type) { + case string: + str = arg.(string) + case common.Address: + a := arg.(common.Address) + str = a.String() + case common.Hash: + a := arg.(common.Hash) + str = a.String() + case []byte: + a := arg.([]byte) + str = hexutil.Encode(a) + } + + return +} diff --git a/pkg/omni/shared/contract/contract_test.go b/pkg/omni/shared/contract/contract_test.go index 5d9da792..6cc16387 100644 --- a/pkg/omni/shared/contract/contract_test.go +++ b/pkg/omni/shared/contract/contract_test.go @@ -23,6 +23,7 @@ import ( "github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) var _ = Describe("Contract", func() { @@ -61,45 +62,45 @@ var _ = Describe("Contract", func() { BeforeEach(func() { info = &contract.Contract{} - info.MethodAddrs = map[string]bool{} - info.EventAddrs = map[string]bool{} + info.MethodArgs = map[string]bool{} + info.FilterArgs = map[string]bool{} }) It("Returns true if address is in event address filter list", func() { - info.EventAddrs["testAddress1"] = true - info.EventAddrs["testAddress2"] = true + info.FilterArgs["testAddress1"] = true + info.FilterArgs["testAddress2"] = true - is := info.IsEventAddr("testAddress1") + is := info.WantedEventArg("testAddress1") Expect(is).To(Equal(true)) - is = info.IsEventAddr("testAddress2") + is = info.WantedEventArg("testAddress2") Expect(is).To(Equal(true)) - info.MethodAddrs["testAddress3"] = true - is = info.IsEventAddr("testAddress3") + info.MethodArgs["testAddress3"] = true + is = info.WantedEventArg("testAddress3") Expect(is).To(Equal(false)) }) It("Returns true if event address filter is empty (no filter)", func() { - is := info.IsEventAddr("testAddress1") + is := info.WantedEventArg("testAddress1") Expect(is).To(Equal(true)) - is = info.IsEventAddr("testAddress2") + is = info.WantedEventArg("testAddress2") Expect(is).To(Equal(true)) }) It("Returns false if address is not in event address filter list", func() { - info.EventAddrs["testAddress1"] = true - info.EventAddrs["testAddress2"] = true + info.FilterArgs["testAddress1"] = true + info.FilterArgs["testAddress2"] = true - is := info.IsEventAddr("testAddress3") + is := info.WantedEventArg("testAddress3") Expect(is).To(Equal(false)) }) It("Returns false if event address filter is nil (block all)", func() { - info.EventAddrs = nil + info.FilterArgs = nil - is := info.IsEventAddr("testAddress1") + is := info.WantedEventArg("testAddress1") Expect(is).To(Equal(false)) - is = info.IsEventAddr("testAddress2") + is = info.WantedEventArg("testAddress2") Expect(is).To(Equal(false)) }) }) @@ -107,45 +108,45 @@ var _ = Describe("Contract", func() { Describe("IsMethodAddr", func() { BeforeEach(func() { info = &contract.Contract{} - info.MethodAddrs = map[string]bool{} - info.EventAddrs = map[string]bool{} + info.MethodArgs = map[string]bool{} + info.FilterArgs = map[string]bool{} }) It("Returns true if address is in method address filter list", func() { - info.MethodAddrs["testAddress1"] = true - info.MethodAddrs["testAddress2"] = true + info.MethodArgs["testAddress1"] = true + info.MethodArgs["testAddress2"] = true - is := info.IsMethodAddr("testAddress1") + is := info.WantedMethodArg("testAddress1") Expect(is).To(Equal(true)) - is = info.IsMethodAddr("testAddress2") + is = info.WantedMethodArg("testAddress2") Expect(is).To(Equal(true)) - info.EventAddrs["testAddress3"] = true - is = info.IsMethodAddr("testAddress3") + info.FilterArgs["testAddress3"] = true + is = info.WantedMethodArg("testAddress3") Expect(is).To(Equal(false)) }) It("Returns true if method address filter list is empty (no filter)", func() { - is := info.IsMethodAddr("testAddress1") + is := info.WantedMethodArg("testAddress1") Expect(is).To(Equal(true)) - is = info.IsMethodAddr("testAddress2") + is = info.WantedMethodArg("testAddress2") Expect(is).To(Equal(true)) }) It("Returns false if address is not in method address filter list", func() { - info.MethodAddrs["testAddress1"] = true - info.MethodAddrs["testAddress2"] = true + info.MethodArgs["testAddress1"] = true + info.MethodArgs["testAddress2"] = true - is := info.IsMethodAddr("testAddress3") + is := info.WantedMethodArg("testAddress3") Expect(is).To(Equal(false)) }) It("Returns false if method address filter list is nil (block all)", func() { - info.MethodAddrs = nil + info.MethodArgs = nil - is := info.IsMethodAddr("testAddress1") + is := info.WantedMethodArg("testAddress1") Expect(is).To(Equal(false)) - is = info.IsMethodAddr("testAddress2") + is = info.WantedMethodArg("testAddress2") Expect(is).To(Equal(false)) }) }) @@ -154,14 +155,14 @@ var _ = Describe("Contract", func() { var mapping map[string]string BeforeEach(func() { info = &contract.Contract{} - info.EventAddrs = map[string]bool{} + info.FilterArgs = map[string]bool{} mapping = map[string]string{} }) It("Return true if event log name-value mapping has filtered for address as a value", func() { - info.EventAddrs["testAddress1"] = true - info.EventAddrs["testAddress2"] = true + info.FilterArgs["testAddress1"] = true + info.FilterArgs["testAddress2"] = true mapping["testInputName1"] = "testAddress1" mapping["testInputName2"] = "testAddress2" @@ -181,8 +182,8 @@ var _ = Describe("Contract", func() { }) It("Return false if event log name-value mapping does not have filtered for address as a value", func() { - info.EventAddrs["testAddress1"] = true - info.EventAddrs["testAddress2"] = true + info.FilterArgs["testAddress1"] = true + info.FilterArgs["testAddress2"] = true mapping["testInputName3"] = "testAddress3" @@ -191,7 +192,7 @@ var _ = Describe("Contract", func() { }) It("Return false if event address filter list is nil (block all)", func() { - info.EventAddrs = nil + info.FilterArgs = nil mapping["testInputName1"] = "testAddress1" mapping["testInputName2"] = "testAddress2" @@ -202,32 +203,33 @@ var _ = Describe("Contract", func() { }) }) - Describe("AddTokenHolderAddress", func() { + Describe("AddEmittedAddr", func() { BeforeEach(func() { info = &contract.Contract{} - info.EventAddrs = map[string]bool{} - info.MethodAddrs = map[string]bool{} - info.TknHolderAddrs = map[string]bool{} + info.FilterArgs = map[string]bool{} + info.MethodArgs = map[string]bool{} + info.Methods = map[string]types.Method{} + info.EmittedAddrs = map[interface{}]bool{} }) It("Adds address to list if it is on the method filter address list", func() { - info.MethodAddrs["testAddress2"] = true - info.AddTokenHolderAddress("testAddress2") - b := info.TknHolderAddrs["testAddress2"] + info.MethodArgs["testAddress2"] = true + info.AddEmittedAddr("testAddress2") + b := info.EmittedAddrs["testAddress2"] Expect(b).To(Equal(true)) }) It("Adds address to list if method filter is empty", func() { - info.AddTokenHolderAddress("testAddress2") - b := info.TknHolderAddrs["testAddress2"] + info.AddEmittedAddr("testAddress2") + b := info.EmittedAddrs["testAddress2"] Expect(b).To(Equal(true)) }) It("Does not add address to list if both filters are closed (nil)", func() { - info.EventAddrs = nil // close both - info.MethodAddrs = nil - info.AddTokenHolderAddress("testAddress1") - b := info.TknHolderAddrs["testAddress1"] + info.FilterArgs = nil // close both + info.MethodArgs = nil + info.AddEmittedAddr("testAddress1") + b := info.EmittedAddrs["testAddress1"] Expect(b).To(Equal(false)) }) }) diff --git a/pkg/omni/shared/helpers/test_helpers/database.go b/pkg/omni/shared/helpers/test_helpers/database.go index 65a28034..666ac612 100644 --- a/pkg/omni/shared/helpers/test_helpers/database.go +++ b/pkg/omni/shared/helpers/test_helpers/database.go @@ -136,17 +136,19 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract Expect(err).ToNot(HaveOccurred()) return &contract.Contract{ - Name: "TrueUSD", - Address: constants.TusdContractAddress, - Abi: p.Abi(), - ParsedAbi: p.ParsedAbi(), - StartingBlock: 6194634, - LastBlock: 6507323, - Events: p.GetEvents(wantedEvents), - Methods: p.GetMethods(wantedMethods), - EventAddrs: map[string]bool{}, - MethodAddrs: map[string]bool{}, - TknHolderAddrs: map[string]bool{}, + Name: "TrueUSD", + Address: constants.TusdContractAddress, + Abi: p.Abi(), + ParsedAbi: p.ParsedAbi(), + StartingBlock: 6194634, + LastBlock: 6507323, + Events: p.GetEvents(wantedEvents), + Methods: p.GetMethods(wantedMethods), + MethodArgs: map[string]bool{}, + FilterArgs: map[string]bool{}, + EmittedAddrs: map[interface{}]bool{}, + EmittedBytes: map[interface{}]bool{}, + EmittedHashes: map[interface{}]bool{}, } } @@ -178,10 +180,10 @@ func TearDown(db *postgres.DB) { _, err = tx.Exec(`ALTER TABLE public.checked_headers DROP COLUMN IF EXISTS transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E CASCADE`) + _, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) Expect(err).NotTo(HaveOccurred()) - _, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E CASCADE`) + _, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`) Expect(err).NotTo(HaveOccurred()) err = tx.Commit() diff --git a/pkg/omni/shared/parser/parser.go b/pkg/omni/shared/parser/parser.go index a1cdd3ad..0ca4e7e0 100644 --- a/pkg/omni/shared/parser/parser.go +++ b/pkg/omni/shared/parser/parser.go @@ -17,9 +17,13 @@ package parser import ( + "errors" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" "github.com/vulcanize/vulcanizedb/pkg/geth" + "github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) @@ -30,7 +34,7 @@ type Parser interface { Abi() string ParsedAbi() abi.ABI GetMethods(wanted []string) map[string]types.Method - GetAddrMethods(wanted []string) map[string]types.Method + GetSelectMethods(wanted []string) map[string]types.Method GetEvents(wanted []string) map[string]types.Event } @@ -59,41 +63,47 @@ func (p *parser) ParsedAbi() abi.ABI { // Retrieves and parses the abi string // for the given contract address func (p *parser) Parse(contractAddr string) error { + // If the abi is one our locally stored abis, fetch + // TODO: Allow users to pass abis through config + knownAbi, err := p.lookUp(contractAddr) + if err == nil { + p.abi = knownAbi + p.parsedAbi, err = geth.ParseAbi(knownAbi) + return err + } + // Try getting abi from etherscan abiStr, err := p.client.GetAbi(contractAddr) if err != nil { return err } - + //TODO: Implement other ways to fetch abi p.abi = abiStr p.parsedAbi, err = geth.ParseAbi(abiStr) return err } +func (p *parser) lookUp(contractAddr string) (string, error) { + if v, ok := constants.Abis[common.HexToAddress(contractAddr)]; ok { + return v, nil + } + + return "", errors.New("ABI not present in lookup tabe") +} + // Returns wanted methods, if they meet the criteria, as map of types.Methods // Empty wanted array => all methods that fit are returned // Nil wanted array => no events are returned -func (p *parser) GetAddrMethods(wanted []string) map[string]types.Method { +func (p *parser) GetSelectMethods(wanted []string) map[string]types.Method { addrMethods := map[string]types.Method{} if wanted == nil { - return addrMethods + return nil } for _, m := range p.parsedAbi.Methods { - // Only return methods that have less than 3 inputs, 1 output, and wanted - if len(m.Inputs) < 3 && len(m.Outputs) == 1 && (len(wanted) == 0 || stringInSlice(wanted, m.Name)) { - addrsOnly := true - for _, input := range m.Inputs { - if input.Type.T != abi.AddressTy { - addrsOnly = false - } - } - - // Only return methods if inputs are all of type address and output is of the accepted types - if addrsOnly && wantType(m.Outputs[0]) { - method := types.NewMethod(m) - addrMethods[method.Name] = method - } + if okInputTypes(m, wanted) { + wantedMethod := types.NewMethod(m) + addrMethods[wantedMethod.Name] = wantedMethod } } @@ -138,9 +148,20 @@ func (p *parser) GetEvents(wanted []string) map[string]types.Event { return events } -func wantType(arg abi.Argument) bool { - wanted := []byte{abi.UintTy, abi.IntTy, abi.BoolTy, abi.StringTy, abi.AddressTy, abi.HashTy} - for _, ty := range wanted { +func okReturnType(arg abi.Argument) bool { + wantedTypes := []byte{ + abi.UintTy, + abi.IntTy, + abi.BoolTy, + abi.StringTy, + abi.AddressTy, + abi.HashTy, + abi.BytesTy, + abi.FixedBytesTy, + abi.FixedPointTy, + } + + for _, ty := range wantedTypes { if arg.Type.T == ty { return true } @@ -149,6 +170,27 @@ func wantType(arg abi.Argument) bool { return false } +func okInputTypes(m abi.Method, wanted []string) bool { + // Only return method if it has less than 3 arguments, a single output value, and it is a method we want or we want all methods (empty 'wanted' slice) + if len(m.Inputs) < 3 && len(m.Outputs) == 1 && (len(wanted) == 0 || stringInSlice(wanted, m.Name)) { + // Only return methods if inputs are all of accepted types and output is of the accepted types + if !okReturnType(m.Outputs[0]) { + return false + } + for _, input := range m.Inputs { + switch input.Type.T { + case abi.AddressTy, abi.HashTy, abi.BytesTy, abi.FixedBytesTy: + default: + return false + } + } + + return true + } + + return false +} + func stringInSlice(list []string, s string) bool { for _, b := range list { if b == s { diff --git a/pkg/omni/shared/parser/parser_test.go b/pkg/omni/shared/parser/parser_test.go index 7631294d..55af1973 100644 --- a/pkg/omni/shared/parser/parser_test.go +++ b/pkg/omni/shared/parser/parser_test.go @@ -174,14 +174,14 @@ var _ = Describe("Parser", func() { }) Describe("GetAddrMethods", func() { - It("Parses and returns only methods whose inputs, if any, are all addresses", func() { + It("Parses and returns only methods whose inputs, if any, are all of type address, hash or []byte", func() { contractAddr := "0xDdE2D979e8d39BB8416eAfcFC1758f3CaB2C9C72" err = p.Parse(contractAddr) Expect(err).ToNot(HaveOccurred()) wanted := []string{"isApprovedForAll", "supportsInterface", "getApproved", "totalSupply", "balanceOf"} methods := p.GetMethods(wanted) - selectMethods := p.GetAddrMethods(wanted) + selectMethods := p.GetSelectMethods(wanted) _, ok := selectMethods["totalSupply"] Expect(ok).To(Equal(true)) @@ -199,7 +199,7 @@ var _ = Describe("Parser", func() { Expect(ok).To(Equal(true)) _, ok = selectMethods["supportsInterface"] - Expect(ok).To(Equal(false)) + Expect(ok).To(Equal(true)) _, ok = methods["supportsInterface"] Expect(ok).To(Equal(true)) diff --git a/pkg/omni/shared/poller/poller.go b/pkg/omni/shared/poller/poller.go index bb8239e9..7e5d43a9 100644 --- a/pkg/omni/shared/poller/poller.go +++ b/pkg/omni/shared/poller/poller.go @@ -22,7 +22,9 @@ import ( "math/big" "strconv" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" @@ -33,6 +35,7 @@ import ( type Poller interface { PollContract(con contract.Contract) error + PollContractAt(con contract.Contract, blockNumber int64) error FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error } @@ -50,22 +53,28 @@ func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *po } } -// Used to call contract's methods found in abi using list of contract-related addresses func (p *poller) PollContract(con contract.Contract) error { + for i := con.StartingBlock; i <= con.LastBlock; i++ { + p.PollContractAt(con, i) + } + + return nil +} + +func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error { p.contract = con - // Iterate over each of the contracts methods for _, m := range con.Methods { switch len(m.Args) { case 0: - if err := p.pollNoArg(m); err != nil { + if err := p.pollNoArgAt(m, blockNumber); err != nil { return err } case 1: - if err := p.pollSingleArg(m); err != nil { + if err := p.pollSingleArgAt(m, blockNumber); err != nil { return err } case 2: - if err := p.pollDoubleArg(m); err != nil { + if err := p.pollDoubleArgAt(m, blockNumber); err != nil { return err } default: @@ -77,61 +86,137 @@ func (p *poller) PollContract(con contract.Contract) error { return nil } -// Poll methods that take no arguments -func (p *poller) pollNoArg(m types.Method) error { +func (p *poller) pollNoArgAt(m types.Method, bn int64) error { result := types.Result{ + Block: bn, Method: m, Inputs: nil, PgType: m.Return[0].PgType, } - for i := p.contract.StartingBlock; i <= p.contract.LastBlock; i++ { - var out interface{} - err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, i) - if err != nil { - return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err)) - } + var out interface{} + err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn) + if err != nil { + return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + } - strOut, err := stringify(out) - if err != nil { - return err - } + strOut, err := stringify(out) + if err != nil { + return err + } - result.Output = strOut - result.Block = i + result.Output = strOut - // Persist result immediately - err = p.PersistResult(result, p.contract.Address, p.contract.Name) - if err != nil { - return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err)) - } + // Persist result immediately + err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name) + if err != nil { + return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } return nil } // Use token holder address to poll methods that take 1 address argument (e.g. balanceOf) -func (p *poller) pollSingleArg(m types.Method) error { +func (p *poller) pollSingleArgAt(m types.Method, bn int64) error { result := types.Result{ + Block: bn, Method: m, Inputs: make([]interface{}, 1), PgType: m.Return[0].PgType, } - for addr := range p.contract.TknHolderAddrs { - for i := p.contract.StartingBlock; i <= p.contract.LastBlock; i++ { - hashArgs := []common.Address{common.HexToAddress(addr)} - in := make([]interface{}, len(hashArgs)) - strIn := make([]interface{}, len(hashArgs)) - for i, s := range hashArgs { - in[i] = s - strIn[i] = s.String() - } + // Depending on the type of the arg choose + // the correct argument set to iterate over + var args map[interface{}]bool + switch m.Args[0].Type.T { + case abi.FixedBytesTy, abi.BytesTy: + args = p.contract.EmittedBytes + case abi.HashTy: + args = p.contract.EmittedHashes + case abi.AddressTy: + args = p.contract.EmittedAddrs + } + if len(args) == 0 { // If we haven't collected any args by now we can't call the method + return nil + } + results := make([]types.Result, 0, len(args)) + + for arg := range args { + in := []interface{}{arg} + strIn := []interface{}{contract.StringifyArg(arg)} + + var out interface{} + err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn) + if err != nil { + return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + } + strOut, err := stringify(out) + if err != nil { + return err + } + + result.Inputs = strIn + result.Output = strOut + results = append(results, result) + } + + // Persist results as batch + err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) + if err != nil { + return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) + } + + return nil +} + +// Use token holder address to poll methods that take 2 address arguments (e.g. allowance) +func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error { + result := types.Result{ + Block: bn, + Method: m, + Inputs: make([]interface{}, 2), + PgType: m.Return[0].PgType, + } + + // Depending on the type of the args choose + // the correct argument sets to iterate over + var firstArgs map[interface{}]bool + switch m.Args[0].Type.T { + case abi.FixedBytesTy, abi.BytesTy: + firstArgs = p.contract.EmittedBytes + case abi.HashTy: + firstArgs = p.contract.EmittedHashes + case abi.AddressTy: + firstArgs = p.contract.EmittedAddrs + } + if len(firstArgs) == 0 { + return nil + } + + var secondArgs map[interface{}]bool + switch m.Args[1].Type.T { + case abi.FixedBytesTy, abi.BytesTy: + secondArgs = p.contract.EmittedBytes + case abi.HashTy: + secondArgs = p.contract.EmittedHashes + case abi.AddressTy: + secondArgs = p.contract.EmittedAddrs + } + if len(secondArgs) == 0 { + return nil + } + + results := make([]types.Result, 0, len(firstArgs)*len(secondArgs)) + + for arg1 := range firstArgs { + for arg2 := range secondArgs { + in := []interface{}{arg1, arg2} + strIn := []interface{}{contract.StringifyArg(arg1), contract.StringifyArg(arg2)} var out interface{} - err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, i) + err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn) if err != nil { - return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err)) + return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } strOut, err := stringify(out) @@ -140,61 +225,15 @@ func (p *poller) pollSingleArg(m types.Method) error { } result.Output = strOut - result.Block = i result.Inputs = strIn + results = append(results, result) - err = p.PersistResult(result, p.contract.Address, p.contract.Name) - if err != nil { - return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err)) - } } } - return nil -} - -// Use token holder address to poll methods that take 2 address arguments (e.g. allowance) -func (p *poller) pollDoubleArg(m types.Method) error { - // For a large block range and address list this will take a really, really long time- maybe we should only do 1 arg methods - result := types.Result{ - Method: m, - Inputs: make([]interface{}, 2), - PgType: m.Return[0].PgType, - } - - for addr1 := range p.contract.TknHolderAddrs { - for addr2 := range p.contract.TknHolderAddrs { - for i := p.contract.StartingBlock; i <= p.contract.LastBlock; i++ { - hashArgs := []common.Address{common.HexToAddress(addr1), common.HexToAddress(addr2)} - in := make([]interface{}, len(hashArgs)) - strIn := make([]interface{}, len(hashArgs)) - for i, s := range hashArgs { - in[i] = s - strIn[i] = s.String() - } - - var out interface{} - err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, i) - if err != nil { - return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err)) - } - - strOut, err := stringify(out) - if err != nil { - return err - } - - result.Output = strOut - result.Block = i - result.Inputs = strIn - - err = p.PersistResult(result, p.contract.Address, p.contract.Name) - if err != nil { - return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err)) - } - } - - } + err := p.PersistResults(results, m, p.contract.Address, p.contract.Name) + if err != nil { + return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)) } return nil @@ -208,19 +247,22 @@ func (p *poller) FetchContractData(contractAbi, contractAddress, method string, func stringify(input interface{}) (string, error) { switch input.(type) { case *big.Int: - var b *big.Int - b = input.(*big.Int) + b := input.(*big.Int) return b.String(), nil case common.Address: - var a common.Address - a = input.(common.Address) + a := input.(common.Address) return a.String(), nil case common.Hash: - var h common.Hash - h = input.(common.Hash) + h := input.(common.Hash) return h.String(), nil case string: return input.(string), nil + case []byte: + b := hexutil.Encode(input.([]byte)) + return b, nil + case byte: + b := input.(byte) + return string(b), nil case bool: return strconv.FormatBool(input.(bool)), nil default: diff --git a/pkg/omni/shared/poller/poller_test.go b/pkg/omni/shared/poller/poller_test.go index a7357a38..438bbae8 100644 --- a/pkg/omni/shared/poller/poller_test.go +++ b/pkg/omni/shared/poller/poller_test.go @@ -19,6 +19,7 @@ package poller_test import ( "fmt" + "github.com/ethereum/go-ethereum/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -54,10 +55,7 @@ var _ = Describe("Poller", func() { Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 con.LastBlock = 6707323 - con.TknHolderAddrs = map[string]bool{ - "0xfE9e8709d3215310075d67E3ed32A380CCf451C8": true, - "0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE": true, - } + con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) err := p.PollContract(*con) Expect(err).ToNot(HaveOccurred()) @@ -90,10 +88,7 @@ var _ = Describe("Poller", func() { Expect(con.Abi).To(Equal(constants.TusdAbiString)) con.StartingBlock = 6707322 con.LastBlock = 6707323 - con.TknHolderAddrs = map[string]bool{ - "0xfE9e8709d3215310075d67E3ed32A380CCf451C8": true, - "0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE": true, - } + con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE")) err := p.PollContract(*con) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/omni/shared/repository/event_repository.go b/pkg/omni/shared/repository/event_repository.go index b733bcb9..b1b4241e 100644 --- a/pkg/omni/shared/repository/event_repository.go +++ b/pkg/omni/shared/repository/event_repository.go @@ -19,12 +19,12 @@ package repository import ( "errors" "fmt" - "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" "strings" "github.com/hashicorp/golang-lru" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" + "github.com/vulcanize/vulcanizedb/pkg/omni/light/repository" "github.com/vulcanize/vulcanizedb/pkg/omni/shared/types" ) @@ -65,12 +65,8 @@ func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository { // Creates table for the watched contract event if needed // Persists converted event log data into this custom table func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error { - if logs == nil { - return errors.New("event repository error: passed a nil log slice") - } - if len(logs) == 0 { - return errors.New("event repository error: passed an empty log slice") + return errors.New("event repository error: passed empty logs slice") } _, err := r.CreateContractSchema(contractAddr) if err != nil { @@ -112,7 +108,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types pgStr = pgStr + "(header_id, token_name, raw_log, log_idx, tx_idx" el := len(event.Values) - // Pack the corresponding variables in a slice + // Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string data := make([]interface{}, 0, 5+el) data = append(data, event.Id, @@ -134,6 +130,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types } pgStr = pgStr + ")" + // Add this query to the transaction _, err = tx.Exec(pgStr, data...) if err != nil { tx.Rollback() @@ -141,8 +138,9 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types } } + // Mark header as checked for this eventId eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr) - err = repository.MarkHeaderCheckedInTransaction(logs[0].Id, tx, eventId) + err = repository.MarkHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block if err != nil { tx.Rollback() return err diff --git a/pkg/omni/shared/repository/event_repository_test.go b/pkg/omni/shared/repository/event_repository_test.go index 6c90711d..6acc6e4d 100644 --- a/pkg/omni/shared/repository/event_repository_test.go +++ b/pkg/omni/shared/repository/event_repository_test.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + "github.com/ethereum/go-ethereum/common" geth "github.com/ethereum/go-ethereum/core/types" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -137,15 +138,15 @@ var _ = Describe("Repository", func() { Expect(err).ToNot(HaveOccurred()) }) - It("Persists contract event log values into custom tables, adding any addresses to a growing list of contract associated addresses", func() { + It("Persists contract event log values into custom tables", func() { err = dataStore.PersistLogs([]types.Log{*log}, event, con.Address, con.Name) Expect(err).ToNot(HaveOccurred()) - b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"] + b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) - b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"] + b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")] Expect(ok).To(Equal(true)) Expect(b).To(Equal(true)) diff --git a/pkg/omni/shared/repository/method_repository.go b/pkg/omni/shared/repository/method_repository.go index 101ffaa2..cace1204 100644 --- a/pkg/omni/shared/repository/method_repository.go +++ b/pkg/omni/shared/repository/method_repository.go @@ -30,8 +30,8 @@ import ( const methodCacheSize = 1000 type MethodRepository interface { - PersistResult(method types.Result, contractAddr, contractName string) error - CreateMethodTable(contractAddr string, method types.Result) (bool, error) + PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error + CreateMethodTable(contractAddr string, method types.Method) (bool, error) CreateContractSchema(contractAddr string) (bool, error) CheckSchemaCache(key string) (interface{}, bool) CheckTableCache(key string) (interface{}, bool) @@ -55,65 +55,74 @@ func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository { } } -func (r *methodRepository) PersistResult(method types.Result, contractAddr, contractName string) error { - if len(method.Args) != len(method.Inputs) { - return errors.New("error: given number of inputs does not match number of method arguments") +// Creates a schema for the contract if needed +// Creates table for the contract method if needed +// Persists method polling data into this custom table +func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { + if len(results) == 0 { + return errors.New("method repository error: passed empty results slice") } - if len(method.Return) != 1 { - return errors.New("error: given number of outputs does not match number of method return values") - } - _, err := r.CreateContractSchema(contractAddr) if err != nil { return err } - _, err = r.CreateMethodTable(contractAddr, method) + _, err = r.CreateMethodTable(contractAddr, methodInfo) if err != nil { return err } - return r.persistResult(method, contractAddr, contractName) + return r.persistResults(results, methodInfo, contractAddr, contractName) } // Creates a custom postgres command to persist logs for the given event -func (r *methodRepository) persistResult(method types.Result, contractAddr, contractName string) error { - // Begin postgres string - pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_method ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name)) - pgStr = pgStr + "(token_name, block" - ml := len(method.Args) - - // Preallocate slice of needed size and proceed to pack variables into it in same order they appear in string - data := make([]interface{}, 0, 3+ml) - data = append(data, - contractName, - method.Block) - - // Iterate over method args and return value, adding names - // to the string and pushing values to the slice - for i, arg := range method.Args { - pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(arg.Name)) // Add underscore after to avoid any collisions with reserved pg words - data = append(data, method.Inputs[i]) - } - pgStr = pgStr + ", returned) VALUES ($1, $2" - data = append(data, method.Output) - - // For each input entry we created we add its postgres command variable to the string - for i := 0; i <= ml; i++ { - pgStr = pgStr + fmt.Sprintf(", $%d", i+3) - } - pgStr = pgStr + ")" - - _, err := r.DB.Exec(pgStr, data...) +func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error { + tx, err := r.DB.Begin() if err != nil { return err } - return nil + for _, result := range results { + // Begin postgres string + pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_method ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(result.Name)) + pgStr = pgStr + "(token_name, block" + ml := len(result.Args) + + // Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string + data := make([]interface{}, 0, 3+ml) + data = append(data, + contractName, + result.Block) + + // Iterate over method args and return value, adding names + // to the string and pushing values to the slice + for i, arg := range result.Args { + pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(arg.Name)) // Add underscore after to avoid any collisions with reserved pg words + data = append(data, result.Inputs[i]) + } + pgStr = pgStr + ", returned) VALUES ($1, $2" + data = append(data, result.Output) + + // For each input entry we created we add its postgres command variable to the string + for i := 0; i <= ml; i++ { + pgStr = pgStr + fmt.Sprintf(", $%d", i+3) + } + pgStr = pgStr + ")" + + // Add this query to the transaction + _, err = tx.Exec(pgStr, data...) + if err != nil { + println("howdy") + tx.Rollback() + return err + } + } + + return tx.Commit() } // Checks for event table and creates it if it does not already exist -func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Result) (bool, error) { +func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) { tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name)) // Check cache before querying pq to see if table exists @@ -139,7 +148,7 @@ func (r *methodRepository) CreateMethodTable(contractAddr string, method types.R } // Creates a table for the given contract and event -func (r *methodRepository) newMethodTable(tableID string, method types.Result) error { +func (r *methodRepository) newMethodTable(tableID string, method types.Method) error { // Begin pg string pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ", tableID) pgStr = pgStr + "(id SERIAL, token_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL," diff --git a/pkg/omni/shared/repository/method_repository_test.go b/pkg/omni/shared/repository/method_repository_test.go index e8bc22bf..4f81fe09 100644 --- a/pkg/omni/shared/repository/method_repository_test.go +++ b/pkg/omni/shared/repository/method_repository_test.go @@ -95,11 +95,11 @@ var _ = Describe("Repository", func() { Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(true)) - created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult) + created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method) Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(true)) - created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult) + created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method) Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(false)) }) @@ -113,7 +113,7 @@ var _ = Describe("Repository", func() { _, ok := dataStore.CheckTableCache(tableID) Expect(ok).To(Equal(false)) - created, err = dataStore.CreateMethodTable(con.Address, mockResult) + created, err = dataStore.CreateMethodTable(con.Address, method) Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(true)) @@ -125,7 +125,7 @@ var _ = Describe("Repository", func() { Describe("PersistResult", func() { It("Persists result from method polling in custom pg table", func() { - err = dataStore.PersistResult(mockResult, con.Address, con.Name) + err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -142,7 +142,7 @@ var _ = Describe("Repository", func() { }) It("Fails with empty result", func() { - err = dataStore.PersistResult(types.Result{}, con.Address, con.Name) + err = dataStore.PersistResults([]types.Result{}, method, con.Address, con.Name) Expect(err).To(HaveOccurred()) }) }) @@ -184,11 +184,11 @@ var _ = Describe("Repository", func() { Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(true)) - created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult) + created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method) Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(true)) - created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult) + created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method) Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(false)) }) @@ -202,7 +202,7 @@ var _ = Describe("Repository", func() { _, ok := dataStore.CheckTableCache(tableID) Expect(ok).To(Equal(false)) - created, err = dataStore.CreateMethodTable(con.Address, mockResult) + created, err = dataStore.CreateMethodTable(con.Address, method) Expect(err).ToNot(HaveOccurred()) Expect(created).To(Equal(true)) @@ -214,7 +214,7 @@ var _ = Describe("Repository", func() { Describe("PersistResult", func() { It("Persists result from method polling in custom pg table for light sync mode vDB", func() { - err = dataStore.PersistResult(mockResult, con.Address, con.Name) + err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name) Expect(err).ToNot(HaveOccurred()) scanStruct := test_helpers.BalanceOf{} @@ -231,7 +231,7 @@ var _ = Describe("Repository", func() { }) It("Fails with empty result", func() { - err = dataStore.PersistResult(types.Result{}, con.Address, con.Name) + err = dataStore.PersistResults([]types.Result{}, method, con.Address, con.Name) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/omni/shared/types/event.go b/pkg/omni/shared/types/event.go index 5a80bb97..f5b5f3d4 100644 --- a/pkg/omni/shared/types/event.go +++ b/pkg/omni/shared/types/event.go @@ -59,7 +59,7 @@ func NewEvent(e abi.Event) Event { fields[i].Indexed = input.Indexed // Fill in pg type based on abi type switch fields[i].Type.T { - case abi.StringTy, abi.HashTy, abi.AddressTy: + case abi.HashTy, abi.AddressTy: fields[i].PgType = "CHARACTER VARYING(66)" case abi.IntTy, abi.UintTy: fields[i].PgType = "DECIMAL" @@ -71,8 +71,6 @@ func NewEvent(e abi.Event) Event { fields[i].PgType = "TEXT[]" case abi.FixedPointTy: fields[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres? - case abi.FunctionTy: - fields[i].PgType = "TEXT" default: fields[i].PgType = "TEXT" } diff --git a/pkg/omni/shared/types/method.go b/pkg/omni/shared/types/method.go index 9c831f79..bced188a 100644 --- a/pkg/omni/shared/types/method.go +++ b/pkg/omni/shared/types/method.go @@ -48,7 +48,7 @@ func NewMethod(m abi.Method) Method { inputs[i].Type = input.Type inputs[i].Indexed = input.Indexed switch inputs[i].Type.T { - case abi.StringTy, abi.HashTy, abi.AddressTy: + case abi.HashTy, abi.AddressTy: inputs[i].PgType = "CHARACTER VARYING(66)" case abi.IntTy, abi.UintTy: inputs[i].PgType = "DECIMAL" @@ -60,8 +60,6 @@ func NewMethod(m abi.Method) Method { inputs[i].PgType = "TEXT[]" case abi.FixedPointTy: inputs[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres? - case abi.FunctionTy: - inputs[i].PgType = "TEXT" default: inputs[i].PgType = "TEXT" } @@ -74,7 +72,7 @@ func NewMethod(m abi.Method) Method { outputs[i].Type = output.Type outputs[i].Indexed = output.Indexed switch outputs[i].Type.T { - case abi.StringTy, abi.HashTy, abi.AddressTy: + case abi.HashTy, abi.AddressTy: outputs[i].PgType = "CHARACTER VARYING(66)" case abi.IntTy, abi.UintTy: outputs[i].PgType = "DECIMAL" @@ -86,8 +84,6 @@ func NewMethod(m abi.Method) Method { outputs[i].PgType = "TEXT[]" case abi.FixedPointTy: outputs[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres? - case abi.FunctionTy: - outputs[i].PgType = "TEXT" default: outputs[i].PgType = "TEXT" }