deep-stack/ipld-eth-server
6
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

Merge pull request #26 from vulcanize/vdb-398-extract-slow-tests

Browse Source 
Vdb 398 extract slow tests
This commit is contained in:
Rob Mulholand 2019-03-08 10:51:51 -06:00 committed by GitHub
commit 288c5ca69f
42 changed files with 1417 additions and 999 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.private_blockchain_password
View File

@ -1 +0,0 @@
1234

10
.travis.yml
View File

@ -6,26 +6,20 @@ services:
- postgresql
addons:
postgresql: "9.6"
ssh_known_hosts:
- 147.75.96.51
go_import_path: github.com/vulcanize/vulcanizedb
before_install:
# ginkgo golint dep goose
- echo -e "Host github.com\n\tHostName github.com\n\tUser git\n\tIdentityFile ~/.ssh/id_rsa\n" >> ~/.ssh/config
- make installtools
- bash ./scripts/install-postgres-10.sh
- npm install -g ganache-cli
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
- echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
- sudo apt-get update && sudo apt-get install yarn
before_script:
- go get -u github.com/pressly/sup/cmd/sup
- sudo -u postgres createdb vulcanize_private
- make migrate NAME=vulcanize_private
- bash ./scripts/start_test_chain.sh
- cd postgraphile && yarn
script:
@ -36,7 +30,3 @@ script:
notifications:
email: false
after_script:
- bash ./scripts/stop_test_chain.sh
- bash ./bin/deploy.sh

64
Supfile
View File

@ -1,64 +0,0 @@
---
version: 0.5
env:
VDB_PATH: /root/go_projects/src/github.com/vulcanize/vulcanizedb
VDB_PG_USER: vulcanize
VDB_PG_PW: vulcanize
networks:
staging:
hosts:
- root@147.75.96.51
prod:
hosts:
- root@147.75.197.13
targets:
deploy:
- remove
- transfer
- buildPostgraphile
- buildVDB
- migrate
- lightSync
- postgraphile
commands:
remove:
desc: remove old vulcanizedb
run: rm -rf $VDB_PATH && rm -rf /usr/local/vulcanizedb && mkdir -p $VDB_PATH
transfer:
desc: transfer repo to remote server
upload:
- src: .
dst: $VDB_PATH
migrate:
desc: run migration
run: >
cd $VDB_PATH &&
make installtools &&
cd db/migrations &&
/root/go_projects/bin/goose postgres "postgresql://$(VDB_PG_USER):$(VDB_PG_PW)@127.0.0.1:5432/vulcanize_public?sslmode=disable" up
buildPostgraphile:
desc: build postgraphile app
run: >
cd $VDB_PATH/postgraphile &&
yarn && tsc
buildVDB:
desc: build vulcanizedb
run: >
cd $VDB_PATH &&
GOPATH=$HOME/go_projects go get &&
GOPATH=$HOME/go_projects go build &&
cp -r . /usr/local/vulcanizedb
lightSync:
desc: start vdb light sync
run: >
systemctl daemon-reload &&
sudo systemctl restart vulcanizedb_light_sync.service &&
sudo systemctl restart vulcanizedb_log_sync.service &&
sudo systemctl restart vulcanizedb_recheck_sync.service
postgraphile:
desc: start postgraphile
run: systemctl daemon-reload && sudo systemctl restart postgraphile.service

5
bin/deploy.sh
View File

@ -1,5 +0,0 @@
if [ $TRAVIS_BRANCH == 'staging' ]; then
sup --debug staging deploy
elif [ $TRAVIS_BRANCH == 'master' ]; then
sup --debug prod deploy
fi

275
integration_test/omni_full_transformer_test.go Normal file
View File

@ -0,0 +1,275 @@
package integration
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Omni full transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var blockRepository repositories.BlockRepository
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194633)))
Expect(c.LastBlock).To(Equal(int64(6194634)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
})
Describe("Execute", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.TransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
Expect(log.Block).To(Equal(int64(6194634)))
Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.NewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
Expect(log.Block).To(Equal(int64(6194635)))
Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(3))
b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
})

413
integration_test/omni_light_transformer_test.go Normal file
View File

@ -0,0 +1,413 @@
package integration
import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Omnit light transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository
var headerID, headerID2 int64
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
headerRepository = repositories.NewHeaderRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194632)))
Expect(c.LastBlock).To(Equal(int64(-1)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
})
Describe("Execute- against TrueUSD contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedAddrs)).To(Equal(4))
Expect(len(c.EmittedHashes)).To(Equal(0))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(2))
Expect(len(c.EmittedAddrs)).To(Equal(0))
b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
Expect(ok).To(Equal(false))
})
It("Polls given method using list of collected hashes", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not persist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against both ENS and TrueUSD", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
header4, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header5, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header6, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
headerRepository.CreateOrUpdateHeader(header4)
headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header6)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
newOwnerLog := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
transferLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(transferLog.HeaderID).To(Equal(headerID))
Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(transferLog.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
ens, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
tusd, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(ens.EmittedHashes)).To(Equal(2))
Expect(len(ens.EmittedAddrs)).To(Equal(0))
Expect(len(tusd.EmittedAddrs)).To(Equal(4))
Expect(len(tusd.EmittedHashes)).To(Equal(0))
b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
It("Polls given methods for each contract, using list of collected values", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
owner := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).To(HaveOccurred())
bal := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).ToNot(HaveOccurred())
Expect(bal.Balance).To(Equal("55849938025000000000000"))
Expect(bal.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).To(HaveOccurred())
})
})
})

8
libraries/shared/README.md Normal file
View File

@ -0,0 +1,8 @@
# Shared Tools
## Description
Code that is useful for or used by plugins written on top of VulcanizeDB.
## Note
Much code in this directory may not be used outside of the tests, but don't delete it - it could be used by a plugin.
Renaming and/or deleting functions in this namespace requires a version bump to avoid breaking plugins.

6
libraries/shared/chunker/log_chunker_test.go
View File

@ -58,9 +58,9 @@ var _ = Describe("Log chunker", func() {
Describe("initialisation", func() {
It("creates lookup maps correctly", func() {
Expect(chunker.AddressToNames).To(Equal(map[string][]string{
"0x00000000000000000000000000000000000000a1": []string{"TransformerA"},
"0x00000000000000000000000000000000000000a2": []string{"TransformerA", "TransformerC"},
"0x00000000000000000000000000000000000000b1": []string{"TransformerB"},
"0x00000000000000000000000000000000000000a1": {"TransformerA"},
"0x00000000000000000000000000000000000000a2": {"TransformerA", "TransformerC"},
"0x00000000000000000000000000000000000000b1": {"TransformerB"},
}))
Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{

3
libraries/shared/constants/data.go
View File

@ -17,6 +17,3 @@
package constants
var DataItemLength = 32
// TODO Grab this from DB, since it can change through governance
var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours

15
libraries/shared/repository/repository.go
View File

@ -18,9 +18,9 @@ package repository
import (
"bytes"
"database/sql"
"database/sql/driver"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -35,7 +35,7 @@ func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn str
return err
}
func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
@ -155,14 +155,3 @@ func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.Transfor
return result.String()
}
func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) {
var blockTimestamp int64
err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp)
if err != nil {
return 0, err
}
tic := blockTimestamp + constants.TTL
return tic, nil
}

189
libraries/shared/repository/repository_utility_test.go → libraries/shared/repository/repository_test.go
View File

@ -19,6 +19,7 @@ package repository_test
import (
"fmt"
"math/rand"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -34,7 +35,83 @@ import (
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Repository utilities", func() {
var _ = Describe("Repository", func() {
Describe("MarkHeaderChecked", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
It("marks passed column as checked for passed header", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
err := shared.MarkHeaderChecked(headerID, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
})
Describe("MarkHeaderCheckedInTransaction", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
It("marks passed column as checked for passed header within a passed transaction", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
tx, txErr := db.Beginx()
Expect(txErr).NotTo(HaveOccurred())
err := shared.MarkHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
})
Describe("MissingHeaders", func() {
var (
db *postgres.DB
@ -116,6 +193,84 @@ var _ = Describe("Repository utilities", func() {
})
})
Describe("RecheckHeaders", func() {
var (
checkedHeadersColumn string
db *postgres.DB
headerOneID, headerTwoID, headerThreeID, headerFourID int64
headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
// create header checked column
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
// create headers
headerRepository := repositories.NewHeaderRepository(db)
headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
Expect(headerOneErr).NotTo(HaveOccurred())
headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
Expect(headerTwoErr).NotTo(HaveOccurred())
headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
Expect(headerThreeErr).NotTo(HaveOccurred())
headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
Expect(headerFourErr).NotTo(HaveOccurred())
// mark every header checked at least once, with one fully rechecked (headerThree)
maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
Expect(intConversionErr).NotTo(HaveOccurred())
_, markHeaderOneCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerOneID, maxCheckCount)
Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
_, markHeaderTwoCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerTwoID, maxCheckCount)
Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
_, markHeaderThreeCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerThreeID, maxCheckCount+1)
Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
_, markHeaderFourCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerFourID, maxCheckCount)
Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
Describe("when no ending block number (ending block number == -1)", func() {
It("returns all headers since starting block where checked count is less than cap", func() {
headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
})
})
Describe("when ending block number specified", func() {
It("returns headers between starting and ending block where checked count is less than cap", func() {
headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
})
})
})
Describe("GetCheckedColumnNames", func() {
It("gets the column names from checked_headers", func() {
db := test_config.NewTestDB(test_config.NewTestNode())
@ -162,33 +317,9 @@ var _ = Describe("Repository utilities", func() {
func getExpectedColumnNames() []string {
return []string{
"price_feeds_checked",
"flip_kick_checked",
"frob_checked",
"tend_checked",
"bite_checked",
"dent_checked",
"pit_file_debt_ceiling_checked",
"pit_file_ilk_checked",
"vat_init_checked",
"drip_file_ilk_checked",
"drip_file_repo_checked",
"drip_file_vow_checked",
"deal_checked",
"drip_drip_checked",
"cat_file_chop_lump_checked",
"cat_file_flip_checked",
"cat_file_pit_vow_checked",
"flop_kick_checked",
"vat_move_checked",
"vat_fold_checked",
"vat_heal_checked",
"vat_toll_checked",
"vat_tune_checked",
"vat_grab_checked",
"vat_flux_checked",
"vat_slip_checked",
"vow_flog_checked",
"flap_kick_checked",
"column_1_checked",
"column_2_checked",
"column_3_checked",
"column_4_checked",
}
}

65
libraries/shared/storage/mappings_test.go Normal file
View File

@ -0,0 +1,65 @@
package storage_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
)
var _ = Describe("Mappings", func() {
Describe("GetMapping", func() {
It("returns the storage key for a mapping when passed the mapping's index on the contract and the desired value's key", func() {
// ex. solidity:
// mapping (bytes32 => uint) public amounts
// to access amounts, pass in the index of the mapping on the contract + the bytes32 key for the uint val being looked up
indexOfMappingOnContract := storage.IndexZero
keyForDesiredValueInMapping := "1234567890abcdef"
storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
Describe("GetNestedMapping", func() {
It("returns the storage key for a nested mapping when passed the mapping's index on the contract and the desired value's keys", func() {
// ex. solidity:
// mapping (bytes32 => uint) public amounts
// mapping (address => mapping (uint => bytes32)) public addressNames
// to access addressNames, pass in the index of the mapping on the contract + the address and uint keys for the bytes32 val being looked up
indexOfMappingOnContract := storage.IndexOne
keyForOuterMapping := "1234567890abcdef"
keyForInnerMapping := "123"
storageKey := storage.GetNestedMapping(indexOfMappingOnContract, keyForOuterMapping, keyForInnerMapping)
expectedStorageKey := common.HexToHash("0x82113529f6cd61061d1a6f0de53f2bdd067a1addd3d2b46be50a99abfcdb1661")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
Describe("GetIncrementedKey", func() {
It("returns the storage key for later values sharing an index on the contract with other earlier values", func() {
// ex. solidity:
// mapping (bytes32 => uint) public amounts
// mapping (address => mapping (uint => bytes32)) public addressNames
// struct Data {
// uint256 quantity;
// uint256 quality;
// }
// mapping (bytes32 => Data) public itemData;
// to access quality from itemData, pass in the storage key for the zero-indexed value (quantity) + the number of increments required.
// (For "quality", we must increment the storage key for the corresponding "quantity" by 1).
indexOfMappingOnContract := storage.IndexTwo
keyForDesiredValueInMapping := "1234567890abcdef"
storageKeyForFirstPropertyOnStruct := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
storageKey := storage.GetIncrementedKey(storageKeyForFirstPropertyOnStruct, 1)
expectedStorageKey := common.HexToHash("0x69b38749f0a8ed5d505c8474f7fb62c7828aad8a7627f1c67e07af1d2368cad4")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
})

22
libraries/shared/storage/utils/value_test.go Normal file
View File

@ -0,0 +1,22 @@
package utils_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
var _ = Describe("Storage value metadata getter", func() {
It("returns a storage value metadata instance with corresponding fields assigned", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.Uint256
expectedMetadata := utils.StorageValueMetadata{
Name: metadataName,
Keys: metadataKeys,
Type: metadataType,
}
Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
})
})

2
libraries/shared/test_data/generic.go
View File

@ -49,7 +49,7 @@ var GenericTestConfig = transformer.TransformerConfig{
}
func randomString(length int) string {
var seededRand *rand.Rand = rand.New(
var seededRand = rand.New(
rand.NewSource(time.Now().UnixNano()))
charset := "abcdefghijklmnopqrstuvwxyz1234567890"
b := make([]byte, length)

8
libraries/shared/watcher/storage_watcher.go
View File

@ -50,8 +50,8 @@ func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
for _, initializer := range initializers {
transformer := initializer(watcher.db)
watcher.Transformers[transformer.ContractAddress()] = transformer
storageTransformer := initializer(watcher.db)
watcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
}
}
@ -65,12 +65,12 @@ func (watcher StorageWatcher) Execute() error {
if parseErr != nil {
return parseErr
}
transformer, ok := watcher.Transformers[row.Contract]
storageTransformer, ok := watcher.Transformers[row.Contract]
if !ok {
logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
continue
}
executeErr := transformer.Execute(row)
executeErr := storageTransformer.Execute(row)
if executeErr != nil {
if isKeyNotFound(executeErr) {
queueErr := watcher.Queue.Add(row)

11
pkg/datastore/postgres/repositories/block_repository.go
View File

@ -17,7 +17,6 @@
package repositories
import (
"context"
"database/sql"
"errors"
log "github.com/sirupsen/logrus"
@ -122,7 +121,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) {
var blockId int64
tx, _ := blockRepository.database.BeginTx(context.Background(), nil)
tx, _ := blockRepository.database.Beginx()
err := tx.QueryRow(
`INSERT INTO blocks
(eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
@ -145,7 +144,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
return blockId, nil
}
func (blockRepository BlockRepository) createTransactions(tx *sql.Tx, blockId int64, transactions []core.Transaction) error {
func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockId int64, transactions []core.Transaction) error {
for _, transaction := range transactions {
err := blockRepository.createTransaction(tx, blockId, transaction)
if err != nil {
@ -165,7 +164,7 @@ func nullStringToZero(s string) string {
return s
}
func (blockRepository BlockRepository) createTransaction(tx *sql.Tx, blockId int64, transaction core.Transaction) error {
func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.Transaction) error {
_, err := tx.Exec(
`INSERT INTO transactions
(block_id, hash, nonce, tx_to, tx_from, gaslimit, gasprice, value, input_data)
@ -198,7 +197,7 @@ func hasReceipt(transaction core.Transaction) bool {
return transaction.Receipt.TxHash != ""
}
func (blockRepository BlockRepository) createReceipt(tx *sql.Tx, blockId int64, receipt core.Receipt) (int, error) {
func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) {
//Not currently persisting log bloom filters
var receiptId int
err := tx.QueryRow(
@ -224,7 +223,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
return retrievedBlockHash, blockExists(retrievedBlockHash)
}
func (blockRepository BlockRepository) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error {
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error {
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

3
pkg/datastore/postgres/repositories/logs_repository.go
View File

@ -17,7 +17,6 @@
package repositories
import (
"context"
"github.com/sirupsen/logrus"
"database/sql"
@ -31,7 +30,7 @@ type LogRepository struct {
}
func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error {
tx, _ := logRepository.DB.BeginTx(context.Background(), nil)
tx, _ := logRepository.DB.Beginx()
for _, tlog := range lgs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

10
pkg/datastore/postgres/repositories/receipt_repository.go
View File

@ -17,8 +17,8 @@
package repositories
import (
"context"
"database/sql"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -31,7 +31,7 @@ type ReceiptRepository struct {
}
func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
tx, err := receiptRepository.DB.BeginTx(context.Background(), nil)
tx, err := receiptRepository.DB.Beginx()
if err != nil {
return err
}
@ -53,7 +53,7 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64,
return nil
}
func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, error) {
func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
var receiptId int64
err := tx.QueryRow(
`INSERT INTO receipts
@ -68,7 +68,7 @@ func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, erro
return receiptId, err
}
func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
for _, log := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@ -84,7 +84,7 @@ func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
}
func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
tx, _ := receiptRepository.DB.BeginTx(context.Background(), nil)
tx, _ := receiptRepository.DB.Beginx()
var receiptId int64
err := tx.QueryRow(
`INSERT INTO receipts

3
pkg/fakes/data.go
View File

@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
@ -49,5 +48,3 @@ func GetFakeHeader(blockNumber int64) core.Header {
Timestamp: strconv.FormatInt(fakeTimestamp, 10),
}
}
var FakeHeaderTic = fakeTimestamp + constants.TTL

14
pkg/fakes/mock_filter_repository.go Normal file
View File

@ -0,0 +1,14 @@
package fakes
import "github.com/vulcanize/vulcanizedb/pkg/filters"
type MockFilterRepository struct {
}
func (*MockFilterRepository) CreateFilter(filter filters.LogFilter) error {
return nil
}
func (*MockFilterRepository) GetFilter(name string) (filters.LogFilter, error) {
panic("implement me")
}

15
pkg/fakes/mock_full_block_retriever.go Normal file
View File

@ -0,0 +1,15 @@
package fakes
type MockFullBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
MostRecentBlock int64
}
func (retriever *MockFullBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockFullBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return retriever.MostRecentBlock, nil
}

14
pkg/fakes/mock_light_block_retriever.go Normal file
View File

@ -0,0 +1,14 @@
package fakes
type MockLightBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
}
func (retriever *MockLightBlockRetriever) RetrieveFirstBlock() (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockLightBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return 0, nil
}

42
pkg/fakes/mock_light_header_repository.go Normal file
View File

@ -0,0 +1,42 @@
package fakes
import "github.com/vulcanize/vulcanizedb/pkg/core"
type MockLightHeaderRepository struct {
}
func (*MockLightHeaderRepository) AddCheckColumn(id string) error {
return nil
}
func (*MockLightHeaderRepository) AddCheckColumns(ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) CheckCache(key string) (interface{}, bool) {
panic("implement me")
}

40
pkg/fakes/mock_parser.go Normal file
View File

@ -0,0 +1,40 @@
package fakes
import (
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
type MockParser struct {
AbiToReturn string
EventName string
Event types.Event
}
func (*MockParser) Parse(contractAddr string) error {
return nil
}
func (*MockParser) ParseAbiStr(abiStr string) error {
panic("implement me")
}
func (parser *MockParser) Abi() string {
return parser.AbiToReturn
}
func (*MockParser) ParsedAbi() abi.ABI {
return abi.ABI{}
}
func (*MockParser) GetMethods(wanted []string) []types.Method {
panic("implement me")
}
func (*MockParser) GetSelectMethods(wanted []string) []types.Method {
return []types.Method{}
}
func (parser *MockParser) GetEvents(wanted []string) map[string]types.Event {
return map[string]types.Event{parser.EventName: parser.Event}
}

24
pkg/fakes/mock_poller.go Normal file
View File

@ -0,0 +1,24 @@
package fakes
import (
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
)
type MockPoller struct {
ContractName string
}
func (*MockPoller) PollContract(con contract.Contract) error {
panic("implement me")
}
func (*MockPoller) PollContractAt(con contract.Contract, blockNumber int64) error {
panic("implement me")
}
func (poller *MockPoller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
if p, ok := result.(*string); ok {
*p = poller.ContractName
}
return nil
}

4
pkg/omni/full/converter/converter.go
View File

@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) {
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
values := make(map[string]interface{})
log := helpers.ConvertToLog(watchedEvent)
err := contract.UnpackLogIntoMap(values, event.Name, log)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}

94
pkg/omni/full/transformer/transformer.go
View File

@ -34,7 +34,7 @@ import (
)
// Requires a fully synced vDB and a running eth node (or infura)
type transformer struct {
type Transformer struct {
// Database interfaces
datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters()
datastore.WatchedEventRepository // Watched event log views, created by the log filters
@ -76,8 +76,8 @@ type transformer struct {
}
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transformer {
return &transformer{
func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer {
return &Transformer{
Poller: poller.NewPoller(BC, DB, types.FullSync),
Parser: parser.NewParser(network),
BlockRetriever: retriever.NewBlockRetriever(DB),
@ -101,59 +101,59 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (t *transformer) Init() error {
for contractAddr, subset := range t.WatchedEvents {
func (transformer *Transformer) Init() error {
for contractAddr, subset := range transformer.WatchedEvents {
// Get Abi
err := t.Parser.Parse(contractAddr)
err := transformer.Parser.Parse(contractAddr)
if err != nil {
return err
}
// Get first block and most recent block number in the header repo
firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr)
if err != nil {
return err
}
lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock()
if err != nil {
return err
}
// Set to specified range if it falls within the bounds
if firstBlock < t.ContractStart[contractAddr] {
firstBlock = t.ContractStart[contractAddr]
if firstBlock < transformer.ContractStart[contractAddr] {
firstBlock = transformer.ContractStart[contractAddr]
}
// Get contract name if it has one
var name = new(string)
t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock)
transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range t.EventArgs[contractAddr] {
for _, arg := range transformer.EventArgs[contractAddr] {
eventArgs[arg] = true
}
methodArgs := map[string]bool{}
for _, arg := range t.MethodArgs[contractAddr] {
for _, arg := range transformer.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := contract.Contract{
Name: *name,
Network: t.Network,
Network: transformer.Network,
Address: contractAddr,
Abi: t.Parser.Abi(),
ParsedAbi: t.Parser.ParsedAbi(),
Abi: transformer.Parser.Abi(),
ParsedAbi: transformer.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: t.Parser.GetEvents(subset),
Methods: t.Parser.GetSelectMethods(t.WantedMethods[contractAddr]),
Events: transformer.Parser.GetEvents(subset),
Methods: transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: t.CreateAddrList[contractAddr],
CreateHashList: t.CreateHashList[contractAddr],
Piping: t.Piping[contractAddr],
CreateAddrList: transformer.CreateAddrList[contractAddr],
CreateHashList: transformer.CreateHashList[contractAddr],
Piping: transformer.Piping[contractAddr],
}.Init()
// Use info to create filters
@ -164,14 +164,14 @@ func (t *transformer) Init() error {
// Iterate over filters and push them to the repo using filter repository interface
for _, filter := range info.Filters {
err = t.CreateFilter(filter)
err = transformer.FilterRepository.CreateFilter(filter)
if err != nil {
return err
}
}
// Store contract info for further processing
t.Contracts[contractAddr] = info
transformer.Contracts[contractAddr] = info
}
return nil
@ -182,18 +182,18 @@ func (t *transformer) Init() error {
// Uses converter to convert logs into custom log type
// Persists converted logs into custuom postgres tables
// Calls selected methods, using token holder address generated during event log conversion
func (tr transformer) Execute() error {
if len(tr.Contracts) == 0 {
func (transformer Transformer) Execute() error {
if len(transformer.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts to work with")
}
// Iterate through all internal contracts
for _, con := range tr.Contracts {
for _, con := range transformer.Contracts {
// Update converter with current contract
tr.Update(con)
transformer.Update(con)
// Iterate through contract filters and get watched event logs
for eventSig, filter := range con.Filters {
watchedEvents, err := tr.GetWatchedEvents(filter.Name)
watchedEvents, err := transformer.GetWatchedEvents(filter.Name)
if err != nil {
return err
}
@ -201,7 +201,7 @@ func (tr transformer) Execute() error {
// Iterate over watched event logs
for _, we := range watchedEvents {
// Convert them to our custom log type
cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig])
if err != nil {
return err
}
@ -211,7 +211,7 @@ func (tr transformer) Execute() error {
// If log is not empty, immediately persist in repo
// Run this in seperate goroutine?
err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
if err != nil {
return err
}
@ -222,7 +222,7 @@ func (tr transformer) Execute() error {
// poller polls select contract methods
// and persists the results into custom pg tables
// Run this in seperate goroutine?
if err := tr.PollContract(*con); err != nil {
if err := transformer.PollContract(*con); err != nil {
return err
}
}
@ -231,41 +231,41 @@ func (tr transformer) Execute() error {
}
// Used to set which contract addresses and which of their events to watch
func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) {
transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to watch events for
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) {
transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
tr.ContractStart[strings.ToLower(contractAddr)] = start
func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) {
transformer.ContractStart[strings.ToLower(contractAddr)] = start
}
// Used to set whether or not to persist an account address list
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[strings.ToLower(contractAddr)] = on
func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) {
transformer.CreateAddrList[strings.ToLower(contractAddr)] = on
}
// Used to set whether or not to persist an hash list
func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
tr.CreateHashList[strings.ToLower(contractAddr)] = on
func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) {
transformer.CreateHashList[strings.ToLower(contractAddr)] = on
}
// Used to turn method piping on for a contract
func (tr *transformer) SetPiping(contractAddr string, on bool) {
tr.Piping[strings.ToLower(contractAddr)] = on
func (transformer *Transformer) SetPiping(contractAddr string, on bool) {
transformer.Piping[strings.ToLower(contractAddr)] = on
}

367
pkg/omni/full/transformer/transformer_test.go
View File

@ -17,353 +17,160 @@
package transformer_test
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
var _ = Describe("Transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var blockRepository repositories.BlockRepository
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
var fakeAddress = "0x1234567890abcdef"
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("SetEvents", func() {
It("Sets which events to watch from the given contract address", func() {
watchedEvents := []string{"Transfer", "Mint"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, watchedEvents)
Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, watchedEvents)
Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
})
})
Describe("SetEventAddrs", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEventArgs(fakeAddress, eventAddrs)
Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
})
})
Describe("SetMethods", func() {
It("Sets which methods to poll at the given contract address", func() {
watchedMethods := []string{"balanceOf", "totalSupply"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethods(constants.TusdContractAddress, watchedMethods)
Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethods(fakeAddress, watchedMethods)
Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
})
})
Describe("SetMethodAddrs", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethodArgs(fakeAddress, methodAddrs)
Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
})
})
Describe("SetStartingBlock", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetStartingBlock(constants.TusdContractAddress, 11)
Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetStartingBlock(fakeAddress, 11)
Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
})
})
Describe("SetCreateAddrList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateAddrList(constants.TusdContractAddress, true)
Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateAddrList(fakeAddress, true)
Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
})
})
Describe("SetCreateHashList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateHashList(constants.TusdContractAddress, true)
Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateHashList(fakeAddress, true)
Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
})
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockFullBlockRetriever{}
firstBlock := int64(1)
mostRecentBlock := int64(2)
blockRetriever.FirstBlock = firstBlock
blockRetriever.MostRecentBlock = mostRecentBlock
parsr := &fakes.MockParser{}
fakeAbi := "fake_abi"
eventName := "Transfer"
event := types.Event{}
parsr.AbiToReturn = fakeAbi
parsr.EventName = eventName
parsr.Event = event
pollr := &fakes.MockPoller{}
fakeContractName := "fake_contract_name"
pollr.ContractName = fakeContractName
t := getTransformer(blockRetriever, parsr, pollr)
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
c, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194633)))
Expect(c.LastBlock).To(Equal(int64(6194634)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
Expect(c.StartingBlock).To(Equal(firstBlock))
Expect(c.LastBlock).To(Equal(mostRecentBlock))
Expect(c.Abi).To(Equal(fakeAbi))
Expect(c.Name).To(Equal(fakeContractName))
Expect(c.Address).To(Equal(fakeAddress))
})
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockFullBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("Does nothing if watched events are unset", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
err = t.Init()
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
_, ok := t.Contracts[tusdAddr]
_, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(false))
})
})
Describe("Execute", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.TransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
Expect(log.Block).To(Equal(int64(6194634)))
Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.NewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
Expect(log.Block).To(Equal(int64(6194635)))
Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(3))
b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
})
func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
return transformer.Transformer{
FilterRepository: &fakes.MockFilterRepository{},
Parser: parsr,
BlockRetriever: blockRetriever,
Poller: pollr,
Contracts: map[string]*contract.Contract{},
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
}
}

8
pkg/omni/light/converter/converter.go
View File

@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) {
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
returnLogs := make([]types.Log, 0, len(logs))
for _, log := range logs {
values := make(map[string]interface{})
@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
values[field.Name] = i
}
err := contract.UnpackLogIntoMap(values, event.Name, log)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}
@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
eventsToLogs := make(map[string][]types.Log)
for _, event := range events {
eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
// If the log is of this event type, process it as such
if event.Sig() == log.Topics[0] {
values := make(map[string]interface{})
err := contract.UnpackLogIntoMap(values, event.Name, log)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}

6
pkg/omni/light/repository/header_repository.go
View File

@ -17,8 +17,8 @@
package repository
import (
"database/sql"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/hashicorp/golang-lru"
@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
}
func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
tx, err := r.db.Begin()
tx, err := r.db.Beginx()
if err != nil {
return err
}
@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
return r.columns.Get(key)
}
func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error {
func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO

2
pkg/omni/light/retriever/retriever_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestRetriever(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light BLock Number Retriever Suite Test")
RunSpecs(t, "Light Block Number Retriever Suite Test")
}
var _ = BeforeSuite(func() {

30
pkg/omni/light/transformer/transformer.go
View File

@ -37,7 +37,7 @@ import (
)
// Requires a light synced vDB (headers) and a running eth node (or infura)
type transformer struct {
type Transformer struct {
// Database interfaces
srep.EventRepository // Holds transformed watched event log data
repository.HeaderRepository // Interface for interaction with header repositories
@ -93,9 +93,9 @@ type transformer struct {
// 4. Execute
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transformer {
func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer {
return &transformer{
return &Transformer{
Poller: poller.NewPoller(bc, db, types.LightSync),
Fetcher: fetcher.NewFetcher(bc),
Parser: parser.NewParser(network),
@ -120,7 +120,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (tr *transformer) Init() error {
func (tr *Transformer) Init() error {
// Initialize internally configured transformer settings
tr.contractAddresses = make([]string, 0) // Holds all contract addresses, for batch fetching of logs
tr.sortedEventIds = make(map[string][]string) // Map to sort event column ids by contract, for post fetch processing and persisting of logs
@ -154,7 +154,7 @@ func (tr *transformer) Init() error {
// Get contract name if it has one
var name = new(string)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
@ -221,7 +221,7 @@ func (tr *transformer) Init() error {
return nil
}
func (tr *transformer) Execute() error {
func (tr *Transformer) Execute() error {
if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts")
}
@ -311,7 +311,7 @@ func (tr *transformer) Execute() error {
}
// Used to poll contract methods at a given header
func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
for _, con := range tr.Contracts {
// Skip method polling processes if no methods are specified
// Also don't try to poll methods below this contract's specified starting block
@ -336,41 +336,41 @@ func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[str
}
// Used to set which contract addresses and which of their events to watch
func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) {
tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to watch events for
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) {
tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) {
tr.ContractStart[strings.ToLower(contractAddr)] = start
}
// Used to set whether or not to persist an account address list
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[strings.ToLower(contractAddr)] = on
}
// Used to set whether or not to persist an hash list
func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) {
tr.CreateHashList[strings.ToLower(contractAddr)] = on
}
// Used to turn method piping on for a contract
func (tr *transformer) SetPiping(contractAddr string, on bool) {
func (tr *Transformer) SetPiping(contractAddr string, on bool) {
tr.Piping[strings.ToLower(contractAddr)] = on
}

499
pkg/omni/light/transformer/transformer_test.go
View File

@ -17,491 +17,150 @@
package transformer_test
import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
)
var _ = Describe("Transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository
var headerID, headerID2 int64
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
headerRepository = repositories.NewHeaderRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
var fakeAddress = "0x1234567890abcdef"
Describe("SetEvents", func() {
It("Sets which events to watch from the given contract address", func() {
watchedEvents := []string{"Transfer", "Mint"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, watchedEvents)
Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, watchedEvents)
Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
})
})
Describe("SetEventAddrs", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEventArgs(fakeAddress, eventAddrs)
Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
})
})
Describe("SetMethods", func() {
It("Sets which methods to poll at the given contract address", func() {
watchedMethods := []string{"balanceOf", "totalSupply"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethods(constants.TusdContractAddress, watchedMethods)
Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethods(fakeAddress, watchedMethods)
Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
})
})
Describe("SetMethodAddrs", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethodArgs(fakeAddress, methodAddrs)
Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
})
})
Describe("SetStartingBlock", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetStartingBlock(constants.TusdContractAddress, 11)
Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetStartingBlock(fakeAddress, 11)
Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
})
})
Describe("SetCreateAddrList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateAddrList(constants.TusdContractAddress, true)
Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateAddrList(fakeAddress, true)
Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
})
})
Describe("SetCreateHashList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateHashList(constants.TusdContractAddress, true)
Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateHashList(fakeAddress, true)
Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
})
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockLightBlockRetriever{}
firstBlock := int64(1)
blockRetriever.FirstBlock = firstBlock
parsr := &fakes.MockParser{}
fakeAbi := "fake_abi"
parsr.AbiToReturn = fakeAbi
pollr := &fakes.MockPoller{}
fakeContractName := "fake_contract_name"
pollr.ContractName = fakeContractName
t := getFakeTransformer(blockRetriever, parsr, pollr)
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
c, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194632)))
Expect(c.StartingBlock).To(Equal(firstBlock))
Expect(c.LastBlock).To(Equal(int64(-1)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
Expect(c.Abi).To(Equal(fakeAbi))
Expect(c.Name).To(Equal(fakeContractName))
Expect(c.Address).To(Equal(fakeAddress))
})
It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("Does nothing if watched events are unset", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
err = t.Init()
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
_, ok := t.Contracts[tusdAddr]
_, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(false))
})
})
Describe("Execute- against TrueUSD contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedAddrs)).To(Equal(4))
Expect(len(c.EmittedHashes)).To(Equal(0))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(2))
Expect(len(c.EmittedAddrs)).To(Equal(0))
b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
Expect(ok).To(Equal(false))
})
It("Polls given method using list of collected hashes", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not persist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against both ENS and TrueUSD", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
header4, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header5, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header6, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
headerRepository.CreateOrUpdateHeader(header4)
headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header6)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
newOwnerLog := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
transferLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(transferLog.HeaderID).To(Equal(headerID))
Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(transferLog.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
ens, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
tusd, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(ens.EmittedHashes)).To(Equal(2))
Expect(len(ens.EmittedAddrs)).To(Equal(0))
Expect(len(tusd.EmittedAddrs)).To(Equal(4))
Expect(len(tusd.EmittedHashes)).To(Equal(0))
b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
It("Polls given methods for each contract, using list of collected values", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
owner := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).To(HaveOccurred())
bal := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).ToNot(HaveOccurred())
Expect(bal.Balance).To(Equal("55849938025000000000000"))
Expect(bal.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).To(HaveOccurred())
})
})
})
func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
return transformer.Transformer{
Parser: parsr,
BlockRetriever: blockRetriever,
Poller: pollr,
HeaderRepository: &fakes.MockLightHeaderRepository{},
Contracts: map[string]*contract.Contract{},
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
}
}

1
pkg/omni/shared/constants/constants.go
View File

@ -72,6 +72,7 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b"
var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5"
// TODO: Consider whether these should be moved to plugins
// Contract Owner
var DaiContractOwner = "0x0000000000000000000000000000000000000000"
var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f"

12
pkg/omni/shared/helpers/test_helpers/database.go
View File

@ -106,6 +106,7 @@ type Owner struct {
Address string `db:"returned"`
}
// TODO: consider whether this should be moved to libraries/shared
func SetupBC() core.BlockChain {
infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
rawRpcClient, err := rpc.Dial(infuraIPC)
@ -113,9 +114,9 @@ func SetupBC() core.BlockChain {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
blockChainNode := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
return blockChain
}
@ -127,9 +128,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
blockChainNode := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
@ -188,6 +189,7 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract
}.Init()
}
// TODO: consider whether this can be moved to plugin or libraries/shared
func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) {
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
@ -236,7 +238,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
}
func TearDown(db *postgres.DB) {
tx, err := db.Begin()
tx, err := db.Beginx()
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM blocks`)

4
pkg/omni/shared/repository/event_repository.go
View File

@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
// Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Begin()
tx, err := r.db.Beginx()
if err != nil {
return err
}
@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
// Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Begin()
tx, err := r.db.Beginx()
if err != nil {
return err
}

2
pkg/omni/shared/repository/method_repository.go
View File

@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ
// Creates a custom postgres command to persist logs for the given event
func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
tx, err := r.DB.Begin()
tx, err := r.DB.Beginx()
if err != nil {
return err
}

5
pkg/plugin/test_helpers/database.go
View File

@ -31,6 +31,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
)
// TODO: consider whether this should be moved to libraries/shared
func SetupDBandBC() (*postgres.DB, core.BlockChain) {
infuraIPC := "http://kovan0.vulcanize.io:8545"
rawRpcClient, err := rpc.Dial(infuraIPC)
@ -38,9 +39,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
blockChainNode := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",

2
scripts/start_blockchain
View File

@ -1,2 +0,0 @@
#!/bin/bash
geth --fast --cache=1024

10
scripts/start_test_chain.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
MNEMONIC_PHRASE="whisper ordinary mystery awesome wood fox auction february blind volcano spare soft"
PORT=7545
DATABASE_PATH=test_data/test_chain/
echo Starting ganache chain on port $PORT...
ganache-cli --port $PORT \
--db $DATABASE_PATH \
2>&1 > ganache-output.log &

4
scripts/stop_test_chain.sh
View File

@ -1,4 +0,0 @@
#!/bin/bash
echo "Stopping ganache chain on port 7545"
ps -ef | grep ganache | grep -v grep | awk '{print $2}' | xargs kill