deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

begin work on: Add checked_headers column for methods that are polled so taht we don’t duplicate; Add batching of method polling so that we arent generating a rediculously large account address list before using it to poll methods (or persist the list in pg?); User passed ABI and other ways to get ABI; Add ability to collect []byte and hashes from events and use them in method polling same manner as addresses; Event filter addrs => only those event’s addresses/hashes are used for polling; Option to persist seen address/hash/bytes lists into pg; Only generate lists of addresses, []byte, or hashes if a method will use them later

Browse Source
This commit is contained in:
Ian Norden
2018-12-21 10:33:31 -06:00
parent 8c5b1b4dbe
commit 0a59f06cac
28 changed files with 676 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/omni/shared/contract/contract.go
+97 -23
View File
@@ -20,6 +20,8 @@ import (
"errors"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
@@ -38,13 +40,42 @@ type Contract struct {
ParsedAbi abi.ABI // Parsed abi
Events map[string]types.Event // Map of events to their names
Methods map[string]types.Method // Map of methods to their names
Filters map[string]filters.LogFilter // Map of event filters to their names
EventAddrs map[string]bool // User-input list of account addresses to watch events for
MethodAddrs map[string]bool // User-input list of account addresses to poll methods for
TknHolderAddrs map[string]bool // List of all contract-associated addresses, populated as events are transformed
Filters map[string]filters.LogFilter // Map of event filters to their names; used only for full sync watcher
FilterArgs map[string]bool // User-input list of values to filter event logs for
MethodArgs map[string]bool // User-input list of values to limit method polling to
EmittedAddrs map[interface{}]bool // List of all unique addresses collected from converted event logs
EmittedBytes map[interface{}]bool // List of all unique bytes collected from converted event logs
EmittedHashes map[interface{}]bool // List of all unique hashes collected from converted event logs
CreateAddrList bool // Whether or not to persist address list to postgres
}
// Use contract info to generate event filters
// If we will be calling methods that use addr, hash, or byte arrays
// as arguments then we initialize map to hold these types of values
func (c Contract) Init() *Contract {
for _, method := range c.Methods {
for _, arg := range method.Args {
switch arg.Type.T {
case abi.AddressTy:
c.EmittedAddrs = map[interface{}]bool{}
case abi.HashTy:
c.EmittedHashes = map[interface{}]bool{}
case abi.BytesTy, abi.FixedBytesTy:
c.EmittedBytes = map[interface{}]bool{}
default:
}
}
}
// If we are creating an address list in postgres
// we initialize the map despite what method call, if any
if c.CreateAddrList {
c.EmittedAddrs = map[interface{}]bool{}
}
return &c
}
// Use contract info to generate event filters - full sync omni watcher only
func (c *Contract) GenerateFilters() error {
c.Filters = map[string]filters.LogFilter{}
@@ -65,39 +96,45 @@ func (c *Contract) GenerateFilters() error {
return nil
}
// Returns true if address is in list of addresses to
// Returns true if address is in list of arguments to
// filter events for or if no filtering is specified
func (c *Contract) IsEventAddr(addr string) bool {
if c.EventAddrs == nil {
func (c *Contract) WantedEventArg(arg string) bool {
if c.FilterArgs == nil {
return false
} else if len(c.EventAddrs) == 0 {
} else if len(c.FilterArgs) == 0 {
return true
} else if a, ok := c.EventAddrs[addr]; ok {
} else if a, ok := c.FilterArgs[arg]; ok {
return a
}
return false
}
// Returns true if address is in list of addresses to
// poll methods for or if no filtering is specified
func (c *Contract) IsMethodAddr(addr string) bool {
if c.MethodAddrs == nil {
// Returns true if address is in list of arguments to
// poll methods with or if no filtering is specified
func (c *Contract) WantedMethodArg(arg interface{}) bool {
if c.MethodArgs == nil {
return false
} else if len(c.MethodAddrs) == 0 {
} else if len(c.MethodArgs) == 0 {
return true
} else if a, ok := c.MethodAddrs[addr]; ok {
}
// resolve interface to one of the three types we handle as arguments
str := StringifyArg(arg)
// See if it's hex string has been filtered for
if a, ok := c.MethodArgs[str]; ok {
return a
}
return false
}
// Returns true if mapping value matches filtered for address or if not filter exists
// Returns true if any mapping value matches filtered for address or if no filter exists
// Used to check if an event log name-value mapping should be filtered or not
func (c *Contract) PassesEventFilter(args map[string]string) bool {
for _, arg := range args {
if c.IsEventAddr(arg) {
if c.WantedEventArg(arg) {
return true
}
}
@@ -105,10 +142,47 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool {
return false
}
// Used to add an address to the token holder address list
// if it is on the method polling list or the filter is open
func (c *Contract) AddTokenHolderAddress(addr string) {
if c.TknHolderAddrs != nil && c.IsMethodAddr(addr) {
c.TknHolderAddrs[addr] = true
// Add event emitted address to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
for _, addr := range addresses {
if c.WantedMethodArg(addr) && c.Methods != nil {
c.EmittedAddrs[addr] = true
}
}
}
// Add event emitted hash to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedHash(hashes ...interface{}) {
for _, hash := range hashes {
if c.WantedMethodArg(hash) && c.Methods != nil {
c.EmittedHashes[hash] = true
}
}
}
// Add event emitted bytes to our list if it passes filter and method polling is on
func (c *Contract) AddEmittedBytes(byteArrays ...interface{}) {
for _, bytes := range byteArrays {
if c.WantedMethodArg(bytes) && c.Methods != nil {
c.EmittedBytes[bytes] = true
}
}
}
func StringifyArg(arg interface{}) (str string) {
switch arg.(type) {
case string:
str = arg.(string)
case common.Address:
a := arg.(common.Address)
str = a.String()
case common.Hash:
a := arg.(common.Hash)
str = a.String()
case []byte:
a := arg.([]byte)
str = hexutil.Encode(a)
}
return
}
pkg/omni/shared/contract/contract_test.go
+53 -51
View File
@@ -23,6 +23,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
var _ = Describe("Contract", func() {
@@ -61,45 +62,45 @@ var _ = Describe("Contract", func() {
BeforeEach(func() {
info = &contract.Contract{}
info.MethodAddrs = map[string]bool{}
info.EventAddrs = map[string]bool{}
info.MethodArgs = map[string]bool{}
info.FilterArgs = map[string]bool{}
})
It("Returns true if address is in event address filter list", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
is := info.IsEventAddr("testAddress1")
is := info.WantedEventArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsEventAddr("testAddress2")
is = info.WantedEventArg("testAddress2")
Expect(is).To(Equal(true))
info.MethodAddrs["testAddress3"] = true
is = info.IsEventAddr("testAddress3")
info.MethodArgs["testAddress3"] = true
is = info.WantedEventArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns true if event address filter is empty (no filter)", func() {
is := info.IsEventAddr("testAddress1")
is := info.WantedEventArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsEventAddr("testAddress2")
is = info.WantedEventArg("testAddress2")
Expect(is).To(Equal(true))
})
It("Returns false if address is not in event address filter list", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
is := info.IsEventAddr("testAddress3")
is := info.WantedEventArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns false if event address filter is nil (block all)", func() {
info.EventAddrs = nil
info.FilterArgs = nil
is := info.IsEventAddr("testAddress1")
is := info.WantedEventArg("testAddress1")
Expect(is).To(Equal(false))
is = info.IsEventAddr("testAddress2")
is = info.WantedEventArg("testAddress2")
Expect(is).To(Equal(false))
})
})
@@ -107,45 +108,45 @@ var _ = Describe("Contract", func() {
Describe("IsMethodAddr", func() {
BeforeEach(func() {
info = &contract.Contract{}
info.MethodAddrs = map[string]bool{}
info.EventAddrs = map[string]bool{}
info.MethodArgs = map[string]bool{}
info.FilterArgs = map[string]bool{}
})
It("Returns true if address is in method address filter list", func() {
info.MethodAddrs["testAddress1"] = true
info.MethodAddrs["testAddress2"] = true
info.MethodArgs["testAddress1"] = true
info.MethodArgs["testAddress2"] = true
is := info.IsMethodAddr("testAddress1")
is := info.WantedMethodArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsMethodAddr("testAddress2")
is = info.WantedMethodArg("testAddress2")
Expect(is).To(Equal(true))
info.EventAddrs["testAddress3"] = true
is = info.IsMethodAddr("testAddress3")
info.FilterArgs["testAddress3"] = true
is = info.WantedMethodArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns true if method address filter list is empty (no filter)", func() {
is := info.IsMethodAddr("testAddress1")
is := info.WantedMethodArg("testAddress1")
Expect(is).To(Equal(true))
is = info.IsMethodAddr("testAddress2")
is = info.WantedMethodArg("testAddress2")
Expect(is).To(Equal(true))
})
It("Returns false if address is not in method address filter list", func() {
info.MethodAddrs["testAddress1"] = true
info.MethodAddrs["testAddress2"] = true
info.MethodArgs["testAddress1"] = true
info.MethodArgs["testAddress2"] = true
is := info.IsMethodAddr("testAddress3")
is := info.WantedMethodArg("testAddress3")
Expect(is).To(Equal(false))
})
It("Returns false if method address filter list is nil (block all)", func() {
info.MethodAddrs = nil
info.MethodArgs = nil
is := info.IsMethodAddr("testAddress1")
is := info.WantedMethodArg("testAddress1")
Expect(is).To(Equal(false))
is = info.IsMethodAddr("testAddress2")
is = info.WantedMethodArg("testAddress2")
Expect(is).To(Equal(false))
})
})
@@ -154,14 +155,14 @@ var _ = Describe("Contract", func() {
var mapping map[string]string
BeforeEach(func() {
info = &contract.Contract{}
info.EventAddrs = map[string]bool{}
info.FilterArgs = map[string]bool{}
mapping = map[string]string{}
})
It("Return true if event log name-value mapping has filtered for address as a value", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
mapping["testInputName1"] = "testAddress1"
mapping["testInputName2"] = "testAddress2"
@@ -181,8 +182,8 @@ var _ = Describe("Contract", func() {
})
It("Return false if event log name-value mapping does not have filtered for address as a value", func() {
info.EventAddrs["testAddress1"] = true
info.EventAddrs["testAddress2"] = true
info.FilterArgs["testAddress1"] = true
info.FilterArgs["testAddress2"] = true
mapping["testInputName3"] = "testAddress3"
@@ -191,7 +192,7 @@ var _ = Describe("Contract", func() {
})
It("Return false if event address filter list is nil (block all)", func() {
info.EventAddrs = nil
info.FilterArgs = nil
mapping["testInputName1"] = "testAddress1"
mapping["testInputName2"] = "testAddress2"
@@ -202,32 +203,33 @@ var _ = Describe("Contract", func() {
})
})
Describe("AddTokenHolderAddress", func() {
Describe("AddEmittedAddr", func() {
BeforeEach(func() {
info = &contract.Contract{}
info.EventAddrs = map[string]bool{}
info.MethodAddrs = map[string]bool{}
info.TknHolderAddrs = map[string]bool{}
info.FilterArgs = map[string]bool{}
info.MethodArgs = map[string]bool{}
info.Methods = map[string]types.Method{}
info.EmittedAddrs = map[interface{}]bool{}
})
It("Adds address to list if it is on the method filter address list", func() {
info.MethodAddrs["testAddress2"] = true
info.AddTokenHolderAddress("testAddress2")
b := info.TknHolderAddrs["testAddress2"]
info.MethodArgs["testAddress2"] = true
info.AddEmittedAddr("testAddress2")
b := info.EmittedAddrs["testAddress2"]
Expect(b).To(Equal(true))
})
It("Adds address to list if method filter is empty", func() {
info.AddTokenHolderAddress("testAddress2")
b := info.TknHolderAddrs["testAddress2"]
info.AddEmittedAddr("testAddress2")
b := info.EmittedAddrs["testAddress2"]
Expect(b).To(Equal(true))
})
It("Does not add address to list if both filters are closed (nil)", func() {
info.EventAddrs = nil // close both
info.MethodAddrs = nil
info.AddTokenHolderAddress("testAddress1")
b := info.TknHolderAddrs["testAddress1"]
info.FilterArgs = nil // close both
info.MethodArgs = nil
info.AddEmittedAddr("testAddress1")
b := info.EmittedAddrs["testAddress1"]
Expect(b).To(Equal(false))
})
})