cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

begin work on: Add checked_headers column for methods that are polled so taht we don’t duplicate; Add batching of method polling so that we arent generating a rediculously large account address list before using it to poll methods (or persist the list in pg?); User passed ABI and other ways to get ABI; Add ability to collect []byte and hashes from events and use them in method polling same manner as addresses; Event filter addrs => only those event’s addresses/hashes are used for polling; Option to persist seen address/hash/bytes lists into pg; Only generate lists of addresses, []byte, or hashes if a method will use them later

Browse Source
This commit is contained in:
Ian Norden 2018-12-07 09:38:46 -06:00
parent 8c5b1b4dbe
commit 0a59f06cac
28 changed files with 676 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/lightOmniWatcher.go
View File

@ -67,9 +67,10 @@ func lightOmniWatcher() {
for _, addr := range contractAddresses {
t.SetEvents(addr, contractEvents)
t.SetMethods(addr, contractMethods)
t.SetEventAddrs(addr, eventAddrs)
t.SetMethodAddrs(addr, methodAddrs)
t.SetEventArgs(addr, eventArgs)
t.SetMethodArgs(addr, methodArgs)
t.SetRange(addr, [2]int64{startingBlockNumber, endingBlockNumber})
t.SetCreateAddrList(addr, createAddrList)
}
err := t.Init()
@ -90,11 +91,12 @@ func init() {
lightOmniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
lightOmniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address")
lightOmniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
lightOmniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
lightOmniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses")
lightOmniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses")
lightOmniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched")
lightOmniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
lightOmniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified")
lightOmniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here")
lightOmniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
lightOmniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
lightOmniWatcherCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
lightOmniWatcherCmd.Flags().BoolVarP(&createAddrList, "create-address-list", "c", false, "Set to true to persist address seen in emitted events into the database")
}

13
cmd/omniWatcher.go
View File

@ -67,8 +67,8 @@ func omniWatcher() {
for _, addr := range contractAddresses {
t.SetEvents(addr, contractEvents)
t.SetMethods(addr, contractMethods)
t.SetEventAddrs(addr, eventAddrs)
t.SetMethodAddrs(addr, methodAddrs)
t.SetEventArgs(addr, eventArgs)
t.SetMethodArgs(addr, methodArgs)
t.SetRange(addr, [2]int64{startingBlockNumber, endingBlockNumber})
}
@ -90,11 +90,12 @@ func init() {
omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address")
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
omniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses")
omniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses")
omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "events", "e", []string{}, "Subset of events to watch; by default all events are watched")
omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
omniWatcherCmd.Flags().StringArrayVarP(&eventArgs, "event-args", "f", []string{}, "Argument values to filter event logs for; will only persist event logs that emit at least one of the value specified")
omniWatcherCmd.Flags().StringArrayVarP(&methodArgs, "method-args", "g", []string{}, "Argument values to limit methods to; will only call methods with emitted values that were specified here")
omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
omniWatcherCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
omniWatcherCmd.Flags().BoolVarP(&createAddrList, "create-address-list", "c", false, "Set to true to persist address seen in emitted events into the database")
}

5
cmd/root.go
View File

@ -47,8 +47,9 @@ var (
contractAddresses []string
contractEvents []string
contractMethods []string
eventAddrs []string
methodAddrs []string
eventArgs []string
methodArgs []string
createAddrList bool
)
var rootCmd = &cobra.Command{

45
documentation/ens_questions.md
View File

@ -1,41 +1,46 @@
Run lightSync, starting at the blockheight the ENS registry was published at.
`./vulcanize lightSync --config=./environments/<config.toml> --starting-block-number=3327417`
1. What is the label hash of this domain?
Q. Does this mean for a given namehash of "a.b.c" find keccak256(a), keccak256(b), and keccak256(c)? Do we know the parent domain and/or owner address?
1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract and filter for the root domain and/or owner address to narrow search
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner`
2. For each node + label pair we find emitted, calculate the keccak256(abi.encodePacked(node, label)) and see if it matches our namehash
3. If it does, hash(label) is our answer
Q. Does this mean for a given namehash of "a.b.c" where we don't know what "c" is find keccak256(c)? Do we know the parent domain and/or owner address?
1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract and filter for the parent node and/or owner address to narrow search
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner`
2. For each node + label pair we find emitted, calculate the keccak256(abi.encodePacked(node, label)) and see if it matches our domain's namehash
3. If it does, keccak256(label) is our label hash
2. What is the parent domain of this domain?
1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner`
2. Filter for our label (domain) and collect the node (parent domain namehash) that was emitted with it
3. Call the Registry's resolver(bytes32 node) method for the parent node to find the parent domain's Resolver
4. Call its Resolver's name(bytes32 node) method for the parent node to find the parent domain's name
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner`
2. If we know our label, filter for it and collect the node (parent domain namehash) that was emitted with it
3. If we don't know our label I believe we need to take a similar aqpproach as in section 1 where we try every keccak256(abi.encodePacked(node, label)) until we find the namehash for our domain
4. Call the Registry's resolver(bytes32 node) method for the parent node to find the parent domain's Resolver
5. Call its Resolver's name(bytes32 node) method for the parent node to find the parent domain's name
3. What are the subdomains of this domain?
1. Watch NewOwner events of the ENS Registry contract
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner`
1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) events of the ENS Registry contract
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner`
2. Filter for our node (domain) and collect all the labels emitted with it
3. Calculate subdomain hashes: subnode = keccak256(abi.encodePacked(node, label));
3. Calculate subdomain hashes: subnode = keccak256(abi.encodePacked(node, label))
4. Call the Registry's resolver(bytes32 node) method for a subnode to find the subdomain's Resolver
5. Call its Resolver's name(bytes32 node) method for a subnode to find the subdomain's name
4. What domains does this address own?
1. Watch NewOwner(bytes32 indexed node, bytes32 indexed label, address owner) and Transfer(bytes32 indexed node, address owner) events of the ENS Registry contract and filter for the address
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-eevents=NewOwner --contract-events=Transfer --event-filter-addresses=<address>`
2. Generate list of all nodes this address has ever owned
3. Check which of these they still own at a given blockheight by iterating over the list and calling the owner(bytes32 node) method
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-eevents=NewOwner --contract-events=Transfer --event-filter-addresses=<address>`
2. Collect node and label values; calculate subnodes = keccak256(abi.encodePacked(node, label))
3. Aggregate nodes and subnodes into a list and check which of these they still own at a given blockheight by iterating over the list and calling the Registry's owner(bytes32 node) method
4. Call the Registry's resolver(bytes32 node) method for a node to find the domain's Resolver
5. Call its Resolver's name(bytes32 node) method for the node to find the domain's name
5. What names point to this address?
Q. Is this in terms of which ENS nodes point to a given Resolver address? E.g. All nodes where the ENS records[node].resolver == address? Or is this in terms of Resolver records? E.g. All the records[node].names where the Resolver records[node].addr == address
1. In the former case, watch NewResolver(bytes32 indexed node, address resolver) events of the Registry and filter for the account address
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x1da022710dF5002339274AaDEe8D58218e9D6AB5 --contract-events=NewResolver --event-filter-addresses=<address>`
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewResolver --event-filter-addresses=<address>`
2. Generate a list of nodes that have pointed to this resolver address
3. Check which of these names still point at the address by iterating over the list and calling the resolver(bytes32 node) method
3. Check which of these still point at the address by iterating over the list and calling the Registry's resolver(bytes32 node) method
1. In the latter case, watch AddrChanged(bytes32 indexed node, address a) events of the Resolver and filter for the account address
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x1da022710dF5002339274AaDEe8D58218e9D6AB5 --contract-events=AddrChanged --event-filter-addresses=<address>`
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3648359 --contract-address=0x1da022710dF5002339274AaDEe8D58218e9D6AB5 --contract-events=AddrChanged --event-filter-addresses=<address>`
2. Generate our list of nodes that have pointed towards our address
3. Check which of these they still own at a given blockheight by iterating over the list and calling the Resolver's addr(bytes32 node) method
4. We can then fetch the string names of these nodes using the Resolver's name(bytes32 node) method.
@ -48,9 +53,9 @@ we could also perform []byte filtering on the events and automate polling of eve
currently working on adding this in, and once it is in you would be able to automate more of the steps in these processes.
E.g. you will be able to run
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --contract-events=Transfer --event-args=<address> --contract-methods=owner`
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --contract-events=Transfer --event-args=<address> --contract-methods=owner`
To automate the process in question 4 through step 3 (it will collect node []byte values emitted from the events it watches and then use those to call the owner method, persisting the results)
Or
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=## --ending-block-numer=### --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --event-args=<bytes-to-filter-for>`
`./vulcanize lightOmniWacther --config=./environments/<config.toml> --starting-block-number=3327417 --contract-address=0x314159265dD8dbb310642f98f50C066173C1259b --contract-events=NewOwner --event-args=<bytes-to-filter-for>`
To provide automated filtering for node []byte values in question 3.

19
pkg/omni/full/converter/converter.go
View File

@ -69,7 +69,9 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
}
strValues := make(map[string]string, len(values))
seenBytes := make([]interface{}, 0, len(values))
seenAddrs := make([]interface{}, 0, len(values))
seenHashes := make([]interface{}, 0, len(values))
for fieldName, input := range values {
// Postgres cannot handle custom types, resolve to strings
switch input.(type) {
@ -79,10 +81,11 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
case common.Address:
a := input.(common.Address)
strValues[fieldName] = a.String()
c.ContractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses
seenAddrs = append(seenAddrs, a)
case common.Hash:
h := input.(common.Hash)
strValues[fieldName] = h.String()
seenHashes = append(seenHashes, h)
case string:
strValues[fieldName] = input.(string)
case bool:
@ -90,6 +93,7 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
case []byte:
b := input.([]byte)
strValues[fieldName] = string(b)
seenBytes = append(seenBytes, b)
case byte:
b := input.(byte)
strValues[fieldName] = string(b)
@ -107,6 +111,17 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
Tx: watchedEvent.TxHash,
}
// Cache emitted values if their caching is turned on
if c.ContractInfo.EmittedAddrs != nil {
c.ContractInfo.AddEmittedAddr(seenAddrs...)
}
if c.ContractInfo.EmittedHashes != nil {
c.ContractInfo.AddEmittedHash(seenHashes...)
}
if c.ContractInfo.EmittedBytes != nil {
c.ContractInfo.AddEmittedBytes(seenBytes...)
}
return eventLog, nil
}

10
pkg/omni/full/converter/converter_test.go
View File

@ -81,21 +81,21 @@ var _ = Describe("Converter", func() {
_, err := c.Convert(mocks.MockTranferEvent, event)
Expect(err).ToNot(HaveOccurred())
b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"]
b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = con.TknHolderAddrs["0x"]
_, ok = con.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = con.TknHolderAddrs[""]
_, ok = con.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = con.TknHolderAddrs["0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP"]
_, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})

71
pkg/omni/full/transformer/transformer.go
View File

@ -18,7 +18,7 @@ package transformer
import (
"errors"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -63,8 +63,11 @@ type transformer struct {
// Lists of addresses to filter event or method data
// before persisting; if empty no filter is applied
EventAddrs map[string][]string
MethodAddrs map[string][]string
EventArgs map[string][]string
MethodArgs map[string][]string
// Whether or not to create a list of token holder addresses for the contract in postgres
CreateAddrList map[string]bool
}
// Transformer takes in config for blockchain, database, and network id
@ -81,8 +84,8 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractRanges: map[string][2]int64{},
EventAddrs: map[string][]string{},
MethodAddrs: map[string][]string{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
}
}
@ -116,25 +119,22 @@ func (t *transformer) Init() error {
lastBlock = t.ContractRanges[contractAddr][1]
}
// Get contract name
// Get contract name if it has one
var name = new(string)
err = t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock)
if err != nil {
return errors.New(fmt.Sprintf("unable to fetch contract name: %v\r\n", err))
}
t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock)
// Remove any accidental duplicate inputs in filter addresses
EventAddrs := map[string]bool{}
for _, addr := range t.EventAddrs[contractAddr] {
EventAddrs[addr] = true
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range t.EventArgs[contractAddr] {
eventArgs[arg] = true
}
MethodAddrs := map[string]bool{}
for _, addr := range t.MethodAddrs[contractAddr] {
MethodAddrs[addr] = true
methodArgs := map[string]bool{}
for _, arg := range t.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := &contract.Contract{
info := contract.Contract{
Name: *name,
Network: t.Network,
Address: contractAddr,
@ -143,11 +143,11 @@ func (t *transformer) Init() error {
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: t.GetEvents(subset),
Methods: t.GetAddrMethods(t.WantedMethods[contractAddr]),
EventAddrs: EventAddrs,
MethodAddrs: MethodAddrs,
TknHolderAddrs: map[string]bool{},
}
Methods: t.GetSelectMethods(t.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: t.CreateAddrList[contractAddr],
}.Init()
// Use info to create filters
err = info.GenerateFilters()
@ -222,26 +222,31 @@ func (tr transformer) Execute() error {
}
// Used to set which contract addresses and which of their events to watch
func (t *transformer) SetEvents(contractAddr string, filterSet []string) {
t.WatchedEvents[contractAddr] = filterSet
func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
tr.WatchedEvents[contractAddr] = filterSet
}
// Used to set subset of account addresses to watch events for
func (t *transformer) SetEventAddrs(contractAddr string, filterSet []string) {
t.EventAddrs[contractAddr] = filterSet
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[contractAddr] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (t *transformer) SetMethods(contractAddr string, filterSet []string) {
t.WantedMethods[contractAddr] = filterSet
func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
tr.WantedMethods[contractAddr] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (t *transformer) SetMethodAddrs(contractAddr string, filterSet []string) {
t.MethodAddrs[contractAddr] = filterSet
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[contractAddr] = filterSet
}
// Used to set the block range to watch for a given address
func (t *transformer) SetRange(contractAddr string, rng [2]int64) {
t.ContractRanges[contractAddr] = rng
func (tr *transformer) SetRange(contractAddr string, rng [2]int64) {
tr.ContractRanges[contractAddr] = rng
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[contractAddr] = on
}

34
pkg/omni/full/transformer/transformer_test.go
View File

@ -21,6 +21,7 @@ import (
"math/rand"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -62,8 +63,8 @@ var _ = Describe("Transformer", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventAddrs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventAddrs[constants.TusdContractAddress]).To(Equal(eventAddrs))
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[constants.TusdContractAddress]).To(Equal(eventAddrs))
})
})
@ -80,8 +81,8 @@ var _ = Describe("Transformer", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodAddrs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodAddrs[constants.TusdContractAddress]).To(Equal(methodAddrs))
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[constants.TusdContractAddress]).To(Equal(methodAddrs))
})
})
@ -159,10 +160,10 @@ var _ = Describe("Transformer", func() {
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data", func() {
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -172,18 +173,24 @@ var _ = Describe("Transformer", func() {
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.TknHolderAddrs["0x000000000000000000000000000000000000Af21"]
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843b1234567890"]
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.TknHolderAddrs["0x"]
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
@ -199,7 +206,12 @@ var _ = Describe("Transformer", func() {
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", constants.TusdContractAddress)).StructScan(&res)
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", constants.TusdContractAddress)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", constants.TusdContractAddress)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))

21
pkg/omni/light/converter/converter.go
View File

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
gethTypes "github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
@ -67,6 +68,9 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
}
strValues := make(map[string]string, len(values))
seenBytes := make([]interface{}, 0, len(values))
seenAddrs := make([]interface{}, 0, len(values))
seenHashes := make([]interface{}, 0, len(values))
for fieldName, input := range values {
// Postgres cannot handle custom types, resolve everything to strings
switch input.(type) {
@ -76,17 +80,19 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
case common.Address:
a := input.(common.Address)
strValues[fieldName] = a.String()
c.ContractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses
seenAddrs = append(seenAddrs, a)
case common.Hash:
h := input.(common.Hash)
strValues[fieldName] = h.String()
seenHashes = append(seenHashes, h)
case string:
strValues[fieldName] = input.(string)
case bool:
strValues[fieldName] = strconv.FormatBool(input.(bool))
case []byte:
b := input.([]byte)
strValues[fieldName] = string(b)
strValues[fieldName] = hexutil.Encode(b)
seenBytes = append(seenBytes, b)
case byte:
b := input.(byte)
strValues[fieldName] = string(b)
@ -109,6 +115,17 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
TransactionIndex: log.TxIndex,
Id: headerID,
})
// Cache emitted values if their caching is turned on
if c.ContractInfo.EmittedAddrs != nil {
c.ContractInfo.AddEmittedAddr(seenAddrs...)
}
if c.ContractInfo.EmittedHashes != nil {
c.ContractInfo.AddEmittedHash(seenHashes...)
}
if c.ContractInfo.EmittedBytes != nil {
c.ContractInfo.AddEmittedBytes(seenBytes...)
}
}
}

10
pkg/omni/light/converter/converter_test.go
View File

@ -84,21 +84,21 @@ var _ = Describe("Converter", func() {
_, err := c.Convert([]types.Log{mocks.MockTransferLog1, mocks.MockTransferLog2}, event, 232)
Expect(err).ToNot(HaveOccurred())
b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"]
b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = con.TknHolderAddrs["0x"]
_, ok = con.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = con.TknHolderAddrs[""]
_, ok = con.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = con.TknHolderAddrs["0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP"]
_, ok = con.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})

7
pkg/omni/light/repository/header_repository.go
View File

@ -18,6 +18,7 @@ package repository
import (
"database/sql"
"github.com/hashicorp/golang-lru"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -85,7 +86,8 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE)
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2`
AND headers.eth_node_fingerprint = $2
ORDER BY headers.block_number`
err = r.db.Select(&result, query, startingBlockNumber, r.db.Node.ID)
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
@ -93,7 +95,8 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber int64, endingBlock
WHERE (header_id ISNULL OR ` + eventID + ` IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`
AND headers.eth_node_fingerprint = $3
ORDER BY headers.block_number`
err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
}

2
pkg/omni/light/retriever/block_retriever.go
View File

@ -42,7 +42,7 @@ func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
var firstBlock int
err := r.db.Get(
&firstBlock,
"SELECT block_number FROM headers ORDER BY block_number ASC LIMIT 1",
"SELECT block_number FROM headers ORDER BY block_number LIMIT 1",
)
return int64(firstBlock), err

70
pkg/omni/light/transformer/transformer.go
View File

@ -18,7 +18,6 @@ package transformer
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers"
"strings"
"github.com/ethereum/go-ethereum/common"
@ -30,6 +29,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/omni/light/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
srep "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository"
@ -67,8 +67,11 @@ type transformer struct {
// Lists of addresses to filter event or method data
// before persisting; if empty no filter is applied
EventAddrs map[string][]string
MethodAddrs map[string][]string
EventArgs map[string][]string
MethodArgs map[string][]string
// Whether or not to create a list of token holder addresses for the contract in postgres
CreateAddrList map[string]bool
}
// Transformer takes in config for blockchain, database, and network id
@ -86,8 +89,9 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractRanges: map[string][2]int64{},
EventAddrs: map[string][]string{},
MethodAddrs: map[string][]string{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
}
}
@ -126,18 +130,18 @@ func (tr *transformer) Init() error {
var name = new(string)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
// Remove any potential accidental duplicate inputs in filter addresses
EventAddrs := map[string]bool{}
for _, addr := range tr.EventAddrs[contractAddr] {
EventAddrs[addr] = true
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range tr.EventArgs[contractAddr] {
eventArgs[arg] = true
}
MethodAddrs := map[string]bool{}
for _, addr := range tr.MethodAddrs[contractAddr] {
MethodAddrs[addr] = true
methodArgs := map[string]bool{}
for _, arg := range tr.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := &contract.Contract{
// Aggregate info into contract object and store for execution
tr.Contracts[contractAddr] = contract.Contract{
Name: *name,
Network: tr.Network,
Address: contractAddr,
@ -146,14 +150,11 @@ func (tr *transformer) Init() error {
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: tr.GetEvents(subset),
Methods: tr.GetAddrMethods(tr.WantedMethods[contractAddr]),
EventAddrs: EventAddrs,
MethodAddrs: MethodAddrs,
TknHolderAddrs: map[string]bool{},
}
// Store contract info for execution
tr.Contracts[contractAddr] = info
Methods: tr.GetSelectMethods(tr.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: tr.CreateAddrList[contractAddr],
}.Init()
}
return nil
@ -216,15 +217,15 @@ func (tr *transformer) Execute() error {
if err != nil {
return err
}
}
}
// After persisting all watched event logs
// poller polls select contract methods
// and persists the results into custom pg tables
if err := tr.PollContract(*con); err != nil {
// Poll contract methods at this header's block height
// with arguments collected from event logs up to this point
if err := tr.PollContractAt(*con, header.BlockNumber); err != nil {
return err
}
}
}
}
return nil
}
@ -235,8 +236,8 @@ func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
}
// Used to set subset of account addresses to watch events for
func (tr *transformer) SetEventAddrs(contractAddr string, filterSet []string) {
tr.EventAddrs[contractAddr] = filterSet
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[contractAddr] = filterSet
}
// Used to set which contract addresses and which of their methods to call
@ -245,11 +246,16 @@ func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
}
// Used to set subset of account addresses to poll methods on
func (tr *transformer) SetMethodAddrs(contractAddr string, filterSet []string) {
tr.MethodAddrs[contractAddr] = filterSet
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[contractAddr] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetRange(contractAddr string, rng [2]int64) {
tr.ContractRanges[contractAddr] = rng
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[contractAddr] = on
}

39
pkg/omni/light/transformer/transformer_test.go
View File

@ -18,6 +18,8 @@ package transformer_test
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -59,8 +61,8 @@ var _ = Describe("Transformer", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventAddrs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventAddrs[constants.TusdContractAddress]).To(Equal(eventAddrs))
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[constants.TusdContractAddress]).To(Equal(eventAddrs))
})
})
@ -77,8 +79,8 @@ var _ = Describe("Transformer", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodAddrs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodAddrs[constants.TusdContractAddress]).To(Equal(methodAddrs))
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[constants.TusdContractAddress]).To(Equal(methodAddrs))
})
})
@ -164,10 +166,10 @@ var _ = Describe("Transformer", func() {
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data", func() {
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@ -177,18 +179,24 @@ var _ = Describe("Transformer", func() {
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.TknHolderAddrs["0x1062a747393198f70F71ec65A582423Dba7E5Ab3"]
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.TknHolderAddrs["0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"]
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843b1234567890"]
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.TknHolderAddrs["0x"]
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
@ -204,6 +212,17 @@ var _ = Describe("Transformer", func() {
res := test_helpers.BalanceOf{}
c, ok := t.Contracts[constants.TusdContractAddress]
Expect(ok).To(Equal(true))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", constants.TusdContractAddress)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))

11
pkg/omni/shared/constants/constants.go
View File

File diff suppressed because one or more lines are too long

120
pkg/omni/shared/contract/contract.go
View File

@ -20,6 +20,8 @@ import (
"errors"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
@ -38,13 +40,42 @@ type Contract struct {
ParsedAbi abi.ABI // Parsed abi
Events map[string]types.Event // Map of events to their names
Methods map[string]types.Method // Map of methods to their names
Filters map[string]filters.LogFilter // Map of event filters to their names
EventAddrs map[string]bool // User-input list of account addresses to watch events for
MethodAddrs map[string]bool // User-input list of account addresses to poll methods for
TknHolderAddrs map[string]bool // List of all contract-associated addresses, populated as events are transformed
Filters map[string]filters.LogFilter // Map of event filters to their names; used only for full sync watcher
FilterArgs map[string]bool // User-input list of values to filter event logs for
MethodArgs map[string]bool // User-input list of values to limit method polling to
EmittedAddrs map[interface{}]bool // List of all unique addresses collected from converted event logs
EmittedBytes map[interface{}]bool // List of all unique bytes collected from converted event logs
EmittedHashes map[interface{}]bool // List of all unique hashes collected from converted event logs
CreateAddrList bool // Whether or not to persist address list to postgres
}
// Use contract info to generate event filters
// If we will be calling methods that use addr, hash, or byte arrays
// as arguments then we initialize map to hold these types of values
func (c Contract) Init() *Contract {
for _, method := range c.Methods {
for _, arg := range method.Args {
switch arg.Type.T {
case abi.AddressTy:
c.EmittedAddrs = map[interface{}]bool{}
case abi.HashTy:
c.EmittedHashes = map[interface{}]bool{}
case abi.BytesTy, abi.FixedBytesTy:
c.EmittedBytes = map[interface{}]bool{}
default:
}
}
}
// If we are creating an address list in postgres
// we initialize the map despite what method call, if any
if c.CreateAddrList {
c.EmittedAddrs = map[interface{}]bool{}
}
return &c
}
// Use contract info to generate event filters - full sync omni watcher only
func (c *Contract) GenerateFilters() error {
c.Filters = map[string]filters.LogFilter{}
@ -65,39 +96,45 @@ func (c *Contract) GenerateFilters() error {
return nil
}
// Returns true if address is in list of addresses to
// Returns true if address is in list of arguments to
// filter events for or if no filtering is specified
func (c *Contract) IsEventAddr(addr string) bool {
if c.EventAddrs == nil {
func (c *Contract) WantedEventArg(arg string) bool {
if c.FilterArgs == nil {
return false
} else if len(c.EventAddrs) == 0 {
} else if len(c.FilterArgs) == 0 {
return true
} else if a, ok := c.EventAddrs[addr]; ok {
} else if a, ok := c.FilterArgs[arg]; ok {
return a
}
return false
}
// Returns true if address is in list of addresses to
// poll methods for or if no filtering is specified
func (c *Contract) IsMethodAddr(addr string) bool {
if c.MethodAddrs == nil {
// Returns true if address is in list of arguments to
// poll methods with or if no filtering is specified
func (c *Contract) WantedMethodArg(arg interface{}) bool {
if c.MethodArgs == nil {
return false
} else if len(c.MethodAddrs) == 0 {
} else if len(c.MethodArgs) == 0 {
return true
} else if a, ok := c.MethodAddrs[addr]; ok {
}
// resolve interface to one of the three types we handle as arguments
str := StringifyArg(arg)
// See if it's hex string has been filtered for
if a, ok := c.MethodArgs[str]; ok {
return a
}
return false
}
// Returns true if mapping value matches filtered for address or if not filter exists
// Returns true if any mapping value matches filtered for address or if no filter exists
// Used to check if an event log name-value mapping should be filtered or not
func (c *Contract) PassesEventFilter(args map[string]string) bool {
for _, arg := range args {
if c.IsEventAddr(arg) {
if c.WantedEventArg(arg) {
return true
}
}
@ -105,10 +142,47 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool {
return false
}
// Used to add an address to the token holder address list
// if it is on the method polling list or the filter is open
func (c *Contract) AddTokenHolderAddress(addr string) {
if c.TknHolderAddrs != nil && c.IsMethodAddr(addr) {
c.TknHolderAddrs[addr] = true
// Add event emitted address to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
for _, addr := range addresses {
if c.WantedMethodArg(addr) && c.Methods != nil {
c.EmittedAddrs[addr] = true
}
}
}
// Add event emitted hash to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedHash(hashes ...interface{}) {
for _, hash := range hashes {
if c.WantedMethodArg(hash) && c.Methods != nil {
c.EmittedHashes[hash] = true
}
}
}
// Add event emitted bytes to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedBytes(byteArrays ...interface{}) {
for _, bytes := range byteArrays {
if c.WantedMethodArg(bytes) && c.Methods != nil {
c.EmittedBytes[bytes] = true
}
}
}
func StringifyArg(arg interface{}) (str string) {
switch arg.(type) {
case string:
str = arg.(string)
case common.Address:
a := arg.(common.Address)
str = a.String()
case common.Hash:
a := arg.(common.Hash)
str = a.String()
case []byte:
a := arg.([]byte)
str = hexutil.Encode(a)
}
return
}

104
pkg/omni/shared/contract/contract_test.go
View File

@ -23,6 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
var _ = Describe("Contract", func() {
@ -61,45 +62,45 @@ var _ = Describe("Contract", func() {
BeforeEach(func() {
info = &contract.Contract{}
info.MethodAddrs = map[string]bool{}
info.EventAddrs = map[string]bool{}
info.MethodArgs = map[string]bool{}
info.FilterArgs = map[string]bool{}
})
It("Returns true if address is in event address filter list", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
is := info.IsEventAddr("testAddress1")
is := info.WantedEventArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsEventAddr("testAddress2")
is = info.WantedEventArg("testAddress2")
Expect(is).To(Equal(true))
info.MethodAddrs["testAddress3"] = true
is = info.IsEventAddr("testAddress3")
info.MethodArgs["testAddress3"] = true
is = info.WantedEventArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns true if event address filter is empty (no filter)", func() {
is := info.IsEventAddr("testAddress1")
is := info.WantedEventArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsEventAddr("testAddress2")
is = info.WantedEventArg("testAddress2")
Expect(is).To(Equal(true))
})
It("Returns false if address is not in event address filter list", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
is := info.IsEventAddr("testAddress3")
is := info.WantedEventArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns false if event address filter is nil (block all)", func() {
info.EventAddrs = nil
info.FilterArgs = nil
is := info.IsEventAddr("testAddress1")
is := info.WantedEventArg("testAddress1")
Expect(is).To(Equal(false))
is = info.IsEventAddr("testAddress2")
is = info.WantedEventArg("testAddress2")
Expect(is).To(Equal(false))
})
})
@ -107,45 +108,45 @@ var _ = Describe("Contract", func() {
Describe("IsMethodAddr", func() {
BeforeEach(func() {
info = &contract.Contract{}
info.MethodAddrs = map[string]bool{}
info.EventAddrs = map[string]bool{}
info.MethodArgs = map[string]bool{}
info.FilterArgs = map[string]bool{}
})
It("Returns true if address is in method address filter list", func() {
info.MethodAddrs["testAddress1"] = true
info.MethodAddrs["testAddress2"] = true
info.MethodArgs["testAddress1"] = true
info.MethodArgs["testAddress2"] = true
is := info.IsMethodAddr("testAddress1")
is := info.WantedMethodArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsMethodAddr("testAddress2")
is = info.WantedMethodArg("testAddress2")
Expect(is).To(Equal(true))
info.EventAddrs["testAddress3"] = true
is = info.IsMethodAddr("testAddress3")
info.FilterArgs["testAddress3"] = true
is = info.WantedMethodArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns true if method address filter list is empty (no filter)", func() {
is := info.IsMethodAddr("testAddress1")
is := info.WantedMethodArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsMethodAddr("testAddress2")
is = info.WantedMethodArg("testAddress2")
Expect(is).To(Equal(true))
})
It("Returns false if address is not in method address filter list", func() {
info.MethodAddrs["testAddress1"] = true
info.MethodAddrs["testAddress2"] = true
info.MethodArgs["testAddress1"] = true
info.MethodArgs["testAddress2"] = true
is := info.IsMethodAddr("testAddress3")
is := info.WantedMethodArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns false if method address filter list is nil (block all)", func() {
info.MethodAddrs = nil
info.MethodArgs = nil
is := info.IsMethodAddr("testAddress1")
is := info.WantedMethodArg("testAddress1")
Expect(is).To(Equal(false))
is = info.IsMethodAddr("testAddress2")
is = info.WantedMethodArg("testAddress2")
Expect(is).To(Equal(false))
})
})
@ -154,14 +155,14 @@ var _ = Describe("Contract", func() {
var mapping map[string]string
BeforeEach(func() {
info = &contract.Contract{}
info.EventAddrs = map[string]bool{}
info.FilterArgs = map[string]bool{}
mapping = map[string]string{}
})
It("Return true if event log name-value mapping has filtered for address as a value", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
mapping["testInputName1"] = "testAddress1"
mapping["testInputName2"] = "testAddress2"
@ -181,8 +182,8 @@ var _ = Describe("Contract", func() {
})
It("Return false if event log name-value mapping does not have filtered for address as a value", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
mapping["testInputName3"] = "testAddress3"
@ -191,7 +192,7 @@ var _ = Describe("Contract", func() {
})
It("Return false if event address filter list is nil (block all)", func() {
info.EventAddrs = nil
info.FilterArgs = nil
mapping["testInputName1"] = "testAddress1"
mapping["testInputName2"] = "testAddress2"
@ -202,32 +203,33 @@ var _ = Describe("Contract", func() {
})
})
Describe("AddTokenHolderAddress", func() {
Describe("AddEmittedAddr", func() {
BeforeEach(func() {
info = &contract.Contract{}
info.EventAddrs = map[string]bool{}
info.MethodAddrs = map[string]bool{}
info.TknHolderAddrs = map[string]bool{}
info.FilterArgs = map[string]bool{}
info.MethodArgs = map[string]bool{}
info.Methods = map[string]types.Method{}
info.EmittedAddrs = map[interface{}]bool{}
})
It("Adds address to list if it is on the method filter address list", func() {
info.MethodAddrs["testAddress2"] = true
info.AddTokenHolderAddress("testAddress2")
b := info.TknHolderAddrs["testAddress2"]
info.MethodArgs["testAddress2"] = true
info.AddEmittedAddr("testAddress2")
b := info.EmittedAddrs["testAddress2"]
Expect(b).To(Equal(true))
})
It("Adds address to list if method filter is empty", func() {
info.AddTokenHolderAddress("testAddress2")
b := info.TknHolderAddrs["testAddress2"]
info.AddEmittedAddr("testAddress2")
b := info.EmittedAddrs["testAddress2"]
Expect(b).To(Equal(true))
})
It("Does not add address to list if both filters are closed (nil)", func() {
info.EventAddrs = nil // close both
info.MethodAddrs = nil
info.AddTokenHolderAddress("testAddress1")
b := info.TknHolderAddrs["testAddress1"]
info.FilterArgs = nil // close both
info.MethodArgs = nil
info.AddEmittedAddr("testAddress1")
b := info.EmittedAddrs["testAddress1"]
Expect(b).To(Equal(false))
})
})

12
pkg/omni/shared/helpers/test_helpers/database.go
View File

@ -144,9 +144,11 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract
LastBlock: 6507323,
Events: p.GetEvents(wantedEvents),
Methods: p.GetMethods(wantedMethods),
EventAddrs: map[string]bool{},
MethodAddrs: map[string]bool{},
TknHolderAddrs: map[string]bool{},
MethodArgs: map[string]bool{},
FilterArgs: map[string]bool{},
EmittedAddrs: map[interface{}]bool{},
EmittedBytes: map[interface{}]bool{},
EmittedHashes: map[interface{}]bool{},
}
}
@ -178,10 +180,10 @@ func TearDown(db *postgres.DB) {
_, err = tx.Exec(`ALTER TABLE public.checked_headers DROP COLUMN IF EXISTS transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E CASCADE`)
_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E CASCADE`)
_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
Expect(err).NotTo(HaveOccurred())
err = tx.Commit()

84
pkg/omni/shared/parser/parser.go
View File

@ -17,9 +17,13 @@
package parser
import (
"errors"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/vulcanize/vulcanizedb/pkg/geth"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
@ -30,7 +34,7 @@ type Parser interface {
Abi() string
ParsedAbi() abi.ABI
GetMethods(wanted []string) map[string]types.Method
GetAddrMethods(wanted []string) map[string]types.Method
GetSelectMethods(wanted []string) map[string]types.Method
GetEvents(wanted []string) map[string]types.Event
}
@ -59,41 +63,47 @@ func (p *parser) ParsedAbi() abi.ABI {
// Retrieves and parses the abi string
// for the given contract address
func (p *parser) Parse(contractAddr string) error {
// If the abi is one our locally stored abis, fetch
// TODO: Allow users to pass abis through config
knownAbi, err := p.lookUp(contractAddr)
if err == nil {
p.abi = knownAbi
p.parsedAbi, err = geth.ParseAbi(knownAbi)
return err
}
// Try getting abi from etherscan
abiStr, err := p.client.GetAbi(contractAddr)
if err != nil {
return err
}
//TODO: Implement other ways to fetch abi
p.abi = abiStr
p.parsedAbi, err = geth.ParseAbi(abiStr)
return err
}
func (p *parser) lookUp(contractAddr string) (string, error) {
if v, ok := constants.Abis[common.HexToAddress(contractAddr)]; ok {
return v, nil
}
return "", errors.New("ABI not present in lookup tabe")
}
// Returns wanted methods, if they meet the criteria, as map of types.Methods
// Empty wanted array => all methods that fit are returned
// Nil wanted array => no events are returned
func (p *parser) GetAddrMethods(wanted []string) map[string]types.Method {
func (p *parser) GetSelectMethods(wanted []string) map[string]types.Method {
addrMethods := map[string]types.Method{}
if wanted == nil {
return addrMethods
return nil
}
for _, m := range p.parsedAbi.Methods {
// Only return methods that have less than 3 inputs, 1 output, and wanted
if len(m.Inputs) < 3 && len(m.Outputs) == 1 && (len(wanted) == 0 || stringInSlice(wanted, m.Name)) {
addrsOnly := true
for _, input := range m.Inputs {
if input.Type.T != abi.AddressTy {
addrsOnly = false
}
}
// Only return methods if inputs are all of type address and output is of the accepted types
if addrsOnly && wantType(m.Outputs[0]) {
method := types.NewMethod(m)
addrMethods[method.Name] = method
}
if okInputTypes(m, wanted) {
wantedMethod := types.NewMethod(m)
addrMethods[wantedMethod.Name] = wantedMethod
}
}
@ -138,9 +148,20 @@ func (p *parser) GetEvents(wanted []string) map[string]types.Event {
return events
}
func wantType(arg abi.Argument) bool {
wanted := []byte{abi.UintTy, abi.IntTy, abi.BoolTy, abi.StringTy, abi.AddressTy, abi.HashTy}
for _, ty := range wanted {
func okReturnType(arg abi.Argument) bool {
wantedTypes := []byte{
abi.UintTy,
abi.IntTy,
abi.BoolTy,
abi.StringTy,
abi.AddressTy,
abi.HashTy,
abi.BytesTy,
abi.FixedBytesTy,
abi.FixedPointTy,
}
for _, ty := range wantedTypes {
if arg.Type.T == ty {
return true
}
@ -149,6 +170,27 @@ func wantType(arg abi.Argument) bool {
return false
}
func okInputTypes(m abi.Method, wanted []string) bool {
// Only return method if it has less than 3 arguments, a single output value, and it is a method we want or we want all methods (empty 'wanted' slice)
if len(m.Inputs) < 3 && len(m.Outputs) == 1 && (len(wanted) == 0 || stringInSlice(wanted, m.Name)) {
// Only return methods if inputs are all of accepted types and output is of the accepted types
if !okReturnType(m.Outputs[0]) {
return false
}
for _, input := range m.Inputs {
switch input.Type.T {
case abi.AddressTy, abi.HashTy, abi.BytesTy, abi.FixedBytesTy:
default:
return false
}
}
return true
}
return false
}
func stringInSlice(list []string, s string) bool {
for _, b := range list {
if b == s {

6
pkg/omni/shared/parser/parser_test.go
View File

@ -174,14 +174,14 @@ var _ = Describe("Parser", func() {
})
Describe("GetAddrMethods", func() {
It("Parses and returns only methods whose inputs, if any, are all addresses", func() {
It("Parses and returns only methods whose inputs, if any, are all of type address, hash or []byte", func() {
contractAddr := "0xDdE2D979e8d39BB8416eAfcFC1758f3CaB2C9C72"
err = p.Parse(contractAddr)
Expect(err).ToNot(HaveOccurred())
wanted := []string{"isApprovedForAll", "supportsInterface", "getApproved", "totalSupply", "balanceOf"}
methods := p.GetMethods(wanted)
selectMethods := p.GetAddrMethods(wanted)
selectMethods := p.GetSelectMethods(wanted)
_, ok := selectMethods["totalSupply"]
Expect(ok).To(Equal(true))
@ -199,7 +199,7 @@ var _ = Describe("Parser", func() {
Expect(ok).To(Equal(true))
_, ok = selectMethods["supportsInterface"]
Expect(ok).To(Equal(false))
Expect(ok).To(Equal(true))
_, ok = methods["supportsInterface"]
Expect(ok).To(Equal(true))

158
pkg/omni/shared/poller/poller.go
View File

@ -22,7 +22,9 @@ import (
"math/big"
"strconv"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@ -33,6 +35,7 @@ import (
type Poller interface {
PollContract(con contract.Contract) error
PollContractAt(con contract.Contract, blockNumber int64) error
FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error
}
@ -50,22 +53,28 @@ func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *po
}
}
// Used to call contract's methods found in abi using list of contract-related addresses
func (p *poller) PollContract(con contract.Contract) error {
for i := con.StartingBlock; i <= con.LastBlock; i++ {
p.PollContractAt(con, i)
}
return nil
}
func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error {
p.contract = con
// Iterate over each of the contracts methods
for _, m := range con.Methods {
switch len(m.Args) {
case 0:
if err := p.pollNoArg(m); err != nil {
if err := p.pollNoArgAt(m, blockNumber); err != nil {
return err
}
case 1:
if err := p.pollSingleArg(m); err != nil {
if err := p.pollSingleArgAt(m, blockNumber); err != nil {
return err
}
case 2:
if err := p.pollDoubleArg(m); err != nil {
if err := p.pollDoubleArgAt(m, blockNumber); err != nil {
return err
}
default:
@ -77,19 +86,18 @@ func (p *poller) PollContract(con contract.Contract) error {
return nil
}
// Poll methods that take no arguments
func (p *poller) pollNoArg(m types.Method) error {
func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
result := types.Result{
Block: bn,
Method: m,
Inputs: nil,
PgType: m.Return[0].PgType,
}
for i := p.contract.StartingBlock; i <= p.contract.LastBlock; i++ {
var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, i)
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn)
if err != nil {
return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err))
return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
}
strOut, err := stringify(out)
@ -98,85 +106,117 @@ func (p *poller) pollNoArg(m types.Method) error {
}
result.Output = strOut
result.Block = i
// Persist result immediately
err = p.PersistResult(result, p.contract.Address, p.contract.Name)
err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name)
if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err))
}
return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
}
return nil
}
// Use token holder address to poll methods that take 1 address argument (e.g. balanceOf)
func (p *poller) pollSingleArg(m types.Method) error {
func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
result := types.Result{
Block: bn,
Method: m,
Inputs: make([]interface{}, 1),
PgType: m.Return[0].PgType,
}
for addr := range p.contract.TknHolderAddrs {
for i := p.contract.StartingBlock; i <= p.contract.LastBlock; i++ {
hashArgs := []common.Address{common.HexToAddress(addr)}
in := make([]interface{}, len(hashArgs))
strIn := make([]interface{}, len(hashArgs))
for i, s := range hashArgs {
in[i] = s
strIn[i] = s.String()
// Depending on the type of the arg choose
// the correct argument set to iterate over
var args map[interface{}]bool
switch m.Args[0].Type.T {
case abi.FixedBytesTy, abi.BytesTy:
args = p.contract.EmittedBytes
case abi.HashTy:
args = p.contract.EmittedHashes
case abi.AddressTy:
args = p.contract.EmittedAddrs
}
if len(args) == 0 { // If we haven't collected any args by now we can't call the method
return nil
}
results := make([]types.Result, 0, len(args))
for arg := range args {
in := []interface{}{arg}
strIn := []interface{}{contract.StringifyArg(arg)}
var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, i)
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
if err != nil {
return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err))
return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
}
strOut, err := stringify(out)
if err != nil {
return err
}
result.Output = strOut
result.Block = i
result.Inputs = strIn
result.Output = strOut
results = append(results, result)
}
err = p.PersistResult(result, p.contract.Address, p.contract.Name)
// Persist results as batch
err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err))
}
}
return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
}
return nil
}
// Use token holder address to poll methods that take 2 address arguments (e.g. allowance)
func (p *poller) pollDoubleArg(m types.Method) error {
// For a large block range and address list this will take a really, really long time- maybe we should only do 1 arg methods
func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
result := types.Result{
Block: bn,
Method: m,
Inputs: make([]interface{}, 2),
PgType: m.Return[0].PgType,
}
for addr1 := range p.contract.TknHolderAddrs {
for addr2 := range p.contract.TknHolderAddrs {
for i := p.contract.StartingBlock; i <= p.contract.LastBlock; i++ {
hashArgs := []common.Address{common.HexToAddress(addr1), common.HexToAddress(addr2)}
in := make([]interface{}, len(hashArgs))
strIn := make([]interface{}, len(hashArgs))
for i, s := range hashArgs {
in[i] = s
strIn[i] = s.String()
// Depending on the type of the args choose
// the correct argument sets to iterate over
var firstArgs map[interface{}]bool
switch m.Args[0].Type.T {
case abi.FixedBytesTy, abi.BytesTy:
firstArgs = p.contract.EmittedBytes
case abi.HashTy:
firstArgs = p.contract.EmittedHashes
case abi.AddressTy:
firstArgs = p.contract.EmittedAddrs
}
if len(firstArgs) == 0 {
return nil
}
var secondArgs map[interface{}]bool
switch m.Args[1].Type.T {
case abi.FixedBytesTy, abi.BytesTy:
secondArgs = p.contract.EmittedBytes
case abi.HashTy:
secondArgs = p.contract.EmittedHashes
case abi.AddressTy:
secondArgs = p.contract.EmittedAddrs
}
if len(secondArgs) == 0 {
return nil
}
results := make([]types.Result, 0, len(firstArgs)*len(secondArgs))
for arg1 := range firstArgs {
for arg2 := range secondArgs {
in := []interface{}{arg1, arg2}
strIn := []interface{}{contract.StringifyArg(arg1), contract.StringifyArg(arg2)}
var out interface{}
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, i)
err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
if err != nil {
return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err))
return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
}
strOut, err := stringify(out)
@ -185,16 +225,15 @@ func (p *poller) pollDoubleArg(m types.Method) error {
}
result.Output = strOut
result.Block = i
result.Inputs = strIn
results = append(results, result)
err = p.PersistResult(result, p.contract.Address, p.contract.Name)
}
}
err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
if err != nil {
return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", i, m.Name, p.contract.Address, err))
}
}
}
return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
}
return nil
@ -208,19 +247,22 @@ func (p *poller) FetchContractData(contractAbi, contractAddress, method string,
func stringify(input interface{}) (string, error) {
switch input.(type) {
case *big.Int:
var b *big.Int
b = input.(*big.Int)
b := input.(*big.Int)
return b.String(), nil
case common.Address:
var a common.Address
a = input.(common.Address)
a := input.(common.Address)
return a.String(), nil
case common.Hash:
var h common.Hash
h = input.(common.Hash)
h := input.(common.Hash)
return h.String(), nil
case string:
return input.(string), nil
case []byte:
b := hexutil.Encode(input.([]byte))
return b, nil
case byte:
b := input.(byte)
return string(b), nil
case bool:
return strconv.FormatBool(input.(bool)), nil
default:

11
pkg/omni/shared/poller/poller_test.go
View File

@ -19,6 +19,7 @@ package poller_test
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -54,10 +55,7 @@ var _ = Describe("Poller", func() {
Expect(con.Abi).To(Equal(constants.TusdAbiString))
con.StartingBlock = 6707322
con.LastBlock = 6707323
con.TknHolderAddrs = map[string]bool{
"0xfE9e8709d3215310075d67E3ed32A380CCf451C8": true,
"0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE": true,
}
con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE"))
err := p.PollContract(*con)
Expect(err).ToNot(HaveOccurred())
@ -90,10 +88,7 @@ var _ = Describe("Poller", func() {
Expect(con.Abi).To(Equal(constants.TusdAbiString))
con.StartingBlock = 6707322
con.LastBlock = 6707323
con.TknHolderAddrs = map[string]bool{
"0xfE9e8709d3215310075d67E3ed32A380CCf451C8": true,
"0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE": true,
}
con.AddEmittedAddr(common.HexToAddress("0xfE9e8709d3215310075d67E3ed32A380CCf451C8"), common.HexToAddress("0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE"))
err := p.PollContract(*con)
Expect(err).ToNot(HaveOccurred())

14
pkg/omni/shared/repository/event_repository.go
View File

@ -19,12 +19,12 @@ package repository
import (
"errors"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/repository"
"strings"
"github.com/hashicorp/golang-lru"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
@ -65,12 +65,8 @@ func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
// Creates table for the watched contract event if needed
// Persists converted event log data into this custom table
func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
if logs == nil {
return errors.New("event repository error: passed a nil log slice")
}
if len(logs) == 0 {
return errors.New("event repository error: passed an empty log slice")
return errors.New("event repository error: passed empty logs slice")
}
_, err := r.CreateContractSchema(contractAddr)
if err != nil {
@ -112,7 +108,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
pgStr = pgStr + "(header_id, token_name, raw_log, log_idx, tx_idx"
el := len(event.Values)
// Pack the corresponding variables in a slice
// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
data := make([]interface{}, 0, 5+el)
data = append(data,
event.Id,
@ -134,6 +130,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
}
pgStr = pgStr + ")"
// Add this query to the transaction
_, err = tx.Exec(pgStr, data...)
if err != nil {
tx.Rollback()
@ -141,8 +138,9 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
}
}
// Mark header as checked for this eventId
eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr)
err = repository.MarkHeaderCheckedInTransaction(logs[0].Id, tx, eventId)
err = repository.MarkHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
if err != nil {
tx.Rollback()
return err

7
pkg/omni/shared/repository/event_repository_test.go
View File

@ -21,6 +21,7 @@ import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
geth "github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -137,15 +138,15 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
})
It("Persists contract event log values into custom tables, adding any addresses to a growing list of contract associated addresses", func() {
It("Persists contract event log values into custom tables", func() {
err = dataStore.PersistLogs([]types.Log{*log}, event, con.Address, con.Name)
Expect(err).ToNot(HaveOccurred())
b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"]
b, ok := con.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
b, ok = con.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))

59
pkg/omni/shared/repository/method_repository.go
View File

@ -30,8 +30,8 @@ import (
const methodCacheSize = 1000
type MethodRepository interface {
PersistResult(method types.Result, contractAddr, contractName string) error
CreateMethodTable(contractAddr string, method types.Result) (bool, error)
PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error
CreateMethodTable(contractAddr string, method types.Method) (bool, error)
CreateContractSchema(contractAddr string) (bool, error)
CheckSchemaCache(key string) (interface{}, bool)
CheckTableCache(key string) (interface{}, bool)
@ -55,48 +55,53 @@ func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository {
}
}
func (r *methodRepository) PersistResult(method types.Result, contractAddr, contractName string) error {
if len(method.Args) != len(method.Inputs) {
return errors.New("error: given number of inputs does not match number of method arguments")
// Creates a schema for the contract if needed
// Creates table for the contract method if needed
// Persists method polling data into this custom table
func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
if len(results) == 0 {
return errors.New("method repository error: passed empty results slice")
}
if len(method.Return) != 1 {
return errors.New("error: given number of outputs does not match number of method return values")
}
_, err := r.CreateContractSchema(contractAddr)
if err != nil {
return err
}
_, err = r.CreateMethodTable(contractAddr, method)
_, err = r.CreateMethodTable(contractAddr, methodInfo)
if err != nil {
return err
}
return r.persistResult(method, contractAddr, contractName)
return r.persistResults(results, methodInfo, contractAddr, contractName)
}
// Creates a custom postgres command to persist logs for the given event
func (r *methodRepository) persistResult(method types.Result, contractAddr, contractName string) error {
// Begin postgres string
pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_method ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name))
pgStr = pgStr + "(token_name, block"
ml := len(method.Args)
func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
tx, err := r.DB.Begin()
if err != nil {
return err
}
// Preallocate slice of needed size and proceed to pack variables into it in same order they appear in string
for _, result := range results {
// Begin postgres string
pgStr := fmt.Sprintf("INSERT INTO %s_%s.%s_method ", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(result.Name))
pgStr = pgStr + "(token_name, block"
ml := len(result.Args)
// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
data := make([]interface{}, 0, 3+ml)
data = append(data,
contractName,
method.Block)
result.Block)
// Iterate over method args and return value, adding names
// to the string and pushing values to the slice
for i, arg := range method.Args {
for i, arg := range result.Args {
pgStr = pgStr + fmt.Sprintf(", %s_", strings.ToLower(arg.Name)) // Add underscore after to avoid any collisions with reserved pg words
data = append(data, method.Inputs[i])
data = append(data, result.Inputs[i])
}
pgStr = pgStr + ", returned) VALUES ($1, $2"
data = append(data, method.Output)
data = append(data, result.Output)
// For each input entry we created we add its postgres command variable to the string
for i := 0; i <= ml; i++ {
@ -104,16 +109,20 @@ func (r *methodRepository) persistResult(method types.Result, contractAddr, cont
}
pgStr = pgStr + ")"
_, err := r.DB.Exec(pgStr, data...)
// Add this query to the transaction
_, err = tx.Exec(pgStr, data...)
if err != nil {
println("howdy")
tx.Rollback()
return err
}
}
return nil
return tx.Commit()
}
// Checks for event table and creates it if it does not already exist
func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Result) (bool, error) {
func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) {
tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name))
// Check cache before querying pq to see if table exists
@ -139,7 +148,7 @@ func (r *methodRepository) CreateMethodTable(contractAddr string, method types.R
}
// Creates a table for the given contract and event
func (r *methodRepository) newMethodTable(tableID string, method types.Result) error {
func (r *methodRepository) newMethodTable(tableID string, method types.Method) error {
// Begin pg string
pgStr := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s ", tableID)
pgStr = pgStr + "(id SERIAL, token_name CHARACTER VARYING(66) NOT NULL, block INTEGER NOT NULL,"

20
pkg/omni/shared/repository/method_repository_test.go
View File

@ -95,11 +95,11 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult)
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult)
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(false))
})
@ -113,7 +113,7 @@ var _ = Describe("Repository", func() {
_, ok := dataStore.CheckTableCache(tableID)
Expect(ok).To(Equal(false))
created, err = dataStore.CreateMethodTable(con.Address, mockResult)
created, err = dataStore.CreateMethodTable(con.Address, method)
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
@ -125,7 +125,7 @@ var _ = Describe("Repository", func() {
Describe("PersistResult", func() {
It("Persists result from method polling in custom pg table", func() {
err = dataStore.PersistResult(mockResult, con.Address, con.Name)
err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
Expect(err).ToNot(HaveOccurred())
scanStruct := test_helpers.BalanceOf{}
@ -142,7 +142,7 @@ var _ = Describe("Repository", func() {
})
It("Fails with empty result", func() {
err = dataStore.PersistResult(types.Result{}, con.Address, con.Name)
err = dataStore.PersistResults([]types.Result{}, method, con.Address, con.Name)
Expect(err).To(HaveOccurred())
})
})
@ -184,11 +184,11 @@ var _ = Describe("Repository", func() {
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult)
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, mockResult)
created, err = dataStore.CreateMethodTable(constants.TusdContractAddress, method)
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(false))
})
@ -202,7 +202,7 @@ var _ = Describe("Repository", func() {
_, ok := dataStore.CheckTableCache(tableID)
Expect(ok).To(Equal(false))
created, err = dataStore.CreateMethodTable(con.Address, mockResult)
created, err = dataStore.CreateMethodTable(con.Address, method)
Expect(err).ToNot(HaveOccurred())
Expect(created).To(Equal(true))
@ -214,7 +214,7 @@ var _ = Describe("Repository", func() {
Describe("PersistResult", func() {
It("Persists result from method polling in custom pg table for light sync mode vDB", func() {
err = dataStore.PersistResult(mockResult, con.Address, con.Name)
err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
Expect(err).ToNot(HaveOccurred())
scanStruct := test_helpers.BalanceOf{}
@ -231,7 +231,7 @@ var _ = Describe("Repository", func() {
})
It("Fails with empty result", func() {
err = dataStore.PersistResult(types.Result{}, con.Address, con.Name)
err = dataStore.PersistResults([]types.Result{}, method, con.Address, con.Name)
Expect(err).To(HaveOccurred())
})
})

4
pkg/omni/shared/types/event.go
View File

@ -59,7 +59,7 @@ func NewEvent(e abi.Event) Event {
fields[i].Indexed = input.Indexed
// Fill in pg type based on abi type
switch fields[i].Type.T {
case abi.StringTy, abi.HashTy, abi.AddressTy:
case abi.HashTy, abi.AddressTy:
fields[i].PgType = "CHARACTER VARYING(66)"
case abi.IntTy, abi.UintTy:
fields[i].PgType = "DECIMAL"
@ -71,8 +71,6 @@ func NewEvent(e abi.Event) Event {
fields[i].PgType = "TEXT[]"
case abi.FixedPointTy:
fields[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
case abi.FunctionTy:
fields[i].PgType = "TEXT"
default:
fields[i].PgType = "TEXT"
}

8
pkg/omni/shared/types/method.go
View File

@ -48,7 +48,7 @@ func NewMethod(m abi.Method) Method {
inputs[i].Type = input.Type
inputs[i].Indexed = input.Indexed
switch inputs[i].Type.T {
case abi.StringTy, abi.HashTy, abi.AddressTy:
case abi.HashTy, abi.AddressTy:
inputs[i].PgType = "CHARACTER VARYING(66)"
case abi.IntTy, abi.UintTy:
inputs[i].PgType = "DECIMAL"
@ -60,8 +60,6 @@ func NewMethod(m abi.Method) Method {
inputs[i].PgType = "TEXT[]"
case abi.FixedPointTy:
inputs[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
case abi.FunctionTy:
inputs[i].PgType = "TEXT"
default:
inputs[i].PgType = "TEXT"
}
@ -74,7 +72,7 @@ func NewMethod(m abi.Method) Method {
outputs[i].Type = output.Type
outputs[i].Indexed = output.Indexed
switch outputs[i].Type.T {
case abi.StringTy, abi.HashTy, abi.AddressTy:
case abi.HashTy, abi.AddressTy:
outputs[i].PgType = "CHARACTER VARYING(66)"
case abi.IntTy, abi.UintTy:
outputs[i].PgType = "DECIMAL"
@ -86,8 +84,6 @@ func NewMethod(m abi.Method) Method {
outputs[i].PgType = "TEXT[]"
case abi.FixedPointTy:
outputs[i].PgType = "MONEY" // use shopspring/decimal for fixed point numbers in go and money type in postgres?
case abi.FunctionTy:
outputs[i].PgType = "TEXT"
default:
outputs[i].PgType = "TEXT"
}