deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

begin work on: Add checked_headers column for methods that are polled so taht we don’t duplicate; Add batching of method polling so that we arent generating a rediculously large account address list before using it to poll methods (or persist the list in pg?); User passed ABI and other ways to get ABI; Add ability to collect []byte and hashes from events and use them in method polling same manner as addresses; Event filter addrs => only those event’s addresses/hashes are used for polling; Option to persist seen address/hash/bytes lists into pg; Only generate lists of addresses, []byte, or hashes if a method will use them later

Browse Source
This commit is contained in:
Ian Norden
2018-12-21 10:33:31 -06:00
parent 8c5b1b4dbe
commit 0a59f06cac
28 changed files with 676 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/omni/light/transformer/transformer.go
+38 -32
View File
@@ -18,7 +18,6 @@ package transformer
import (
"errors"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers"
"strings"
"github.com/ethereum/go-ethereum/common"
@@ -30,6 +29,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/omni/light/repository"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
srep "github.com/vulcanize/vulcanizedb/pkg/omni/shared/repository"
@@ -67,8 +67,11 @@ type transformer struct {
// Lists of addresses to filter event or method data
// before persisting; if empty no filter is applied
EventAddrs map[string][]string
MethodAddrs map[string][]string
EventArgs map[string][]string
MethodArgs map[string][]string
// Whether or not to create a list of token holder addresses for the contract in postgres
CreateAddrList map[string]bool
}
// Transformer takes in config for blockchain, database, and network id
@@ -86,8 +89,9 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractRanges: map[string][2]int64{},
EventAddrs: map[string][]string{},
MethodAddrs: map[string][]string{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
}
}
@@ -126,18 +130,18 @@ func (tr *transformer) Init() error {
var name = new(string)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
// Remove any potential accidental duplicate inputs in filter addresses
EventAddrs := map[string]bool{}
for _, addr := range tr.EventAddrs[contractAddr] {
EventAddrs[addr] = true
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range tr.EventArgs[contractAddr] {
eventArgs[arg] = true
}
MethodAddrs := map[string]bool{}
for _, addr := range tr.MethodAddrs[contractAddr] {
MethodAddrs[addr] = true
methodArgs := map[string]bool{}
for _, arg := range tr.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := &contract.Contract{
// Aggregate info into contract object and store for execution
tr.Contracts[contractAddr] = contract.Contract{
Name: *name,
Network: tr.Network,
Address: contractAddr,
@@ -146,14 +150,11 @@ func (tr *transformer) Init() error {
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: tr.GetEvents(subset),
Methods: tr.GetAddrMethods(tr.WantedMethods[contractAddr]),
EventAddrs: EventAddrs,
MethodAddrs: MethodAddrs,
TknHolderAddrs: map[string]bool{},
}
// Store contract info for execution
tr.Contracts[contractAddr] = info
Methods: tr.GetSelectMethods(tr.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: tr.CreateAddrList[contractAddr],
}.Init()
}
return nil
@@ -216,14 +217,14 @@ func (tr *transformer) Execute() error {
if err != nil {
return err
}
// Poll contract methods at this header's block height
// with arguments collected from event logs up to this point
if err := tr.PollContractAt(*con, header.BlockNumber); err != nil {
return err
}
}
}
// After persisting all watched event logs
// poller polls select contract methods
// and persists the results into custom pg tables
if err := tr.PollContract(*con); err != nil {
return err
}
}
return nil
@@ -235,8 +236,8 @@ func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
}
// Used to set subset of account addresses to watch events for
func (tr *transformer) SetEventAddrs(contractAddr string, filterSet []string) {
tr.EventAddrs[contractAddr] = filterSet
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[contractAddr] = filterSet
}
// Used to set which contract addresses and which of their methods to call
@@ -245,11 +246,16 @@ func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
}
// Used to set subset of account addresses to poll methods on
func (tr *transformer) SetMethodAddrs(contractAddr string, filterSet []string) {
tr.MethodAddrs[contractAddr] = filterSet
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[contractAddr] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetRange(contractAddr string, rng [2]int64) {
tr.ContractRanges[contractAddr] = rng
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[contractAddr] = on
}
pkg/omni/light/transformer/transformer_test.go
+29 -10
View File
@@ -18,6 +18,8 @@ package transformer_test
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -59,8 +61,8 @@ var _ = Describe("Transformer", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventAddrs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventAddrs[constants.TusdContractAddress]).To(Equal(eventAddrs))
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[constants.TusdContractAddress]).To(Equal(eventAddrs))
})
})
@@ -77,8 +79,8 @@ var _ = Describe("Transformer", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodAddrs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodAddrs[constants.TusdContractAddress]).To(Equal(methodAddrs))
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[constants.TusdContractAddress]).To(Equal(methodAddrs))
})
})
@@ -164,10 +166,10 @@ var _ = Describe("Transformer", func() {
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data", func() {
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
@@ -177,18 +179,24 @@ var _ = Describe("Transformer", func() {
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.TknHolderAddrs["0x1062a747393198f70F71ec65A582423Dba7E5Ab3"]
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.TknHolderAddrs["0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"]
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843b1234567890"]
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.TknHolderAddrs["0x"]
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
@@ -204,6 +212,17 @@ var _ = Describe("Transformer", func() {
res := test_helpers.BalanceOf{}
c, ok := t.Contracts[constants.TusdContractAddress]
Expect(ok).To(Equal(true))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", constants.TusdContractAddress)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))