cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

Merge branch 'staging' into unpack_update

Browse Source
This commit is contained in:
Ian Norden 2019-03-13 08:16:49 -05:00 committed by GitHub
commit 5dad19b71b
56 changed files with 1845 additions and 1085 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.private_blockchain_password
View File

@ -1 +0,0 @@
1234

10
.travis.yml
View File

@ -6,26 +6,20 @@ services:
- postgresql
addons:
postgresql: "9.6"
ssh_known_hosts:
- 147.75.96.51
go_import_path: github.com/vulcanize/vulcanizedb
before_install:
# ginkgo golint dep goose
- echo -e "Host github.com\n\tHostName github.com\n\tUser git\n\tIdentityFile ~/.ssh/id_rsa\n" >> ~/.ssh/config
- make installtools
- bash ./scripts/install-postgres-10.sh
- npm install -g ganache-cli
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
- echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
- sudo apt-get update && sudo apt-get install yarn
before_script:
- go get -u github.com/pressly/sup/cmd/sup
- sudo -u postgres createdb vulcanize_private
- make migrate NAME=vulcanize_private
- bash ./scripts/start_test_chain.sh
- cd postgraphile && yarn
script:
@ -36,7 +30,3 @@ script:
notifications:
email: false
after_script:
- bash ./scripts/stop_test_chain.sh
- bash ./bin/deploy.sh

23
README.md
View File

@ -31,7 +31,6 @@ Using Vulcanize for the first time requires several steps be done in order to al
In order to fetch the project codebase for local use or modification, install it to your `GOPATH` via:
`go get github.com/vulcanize/vulcanizedb`
`go get gopkg.in/DataDog/dd-trace-go.v1/ddtrace`
Once fetched, dependencies can be installed via `go get` or (the preferred method) at specific versions via `golang/dep`, the prototype golang pakcage manager. Installation instructions are [here](https://golang.github.io/dep/docs/installation.html).
@ -252,8 +251,8 @@ Storage Transformers
* [Example](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/EXAMPLE.md)
Event Transformers
* Guide
* Example
* [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/event_docs/libraries/shared/factories/README.md)
* [Example](https://github.com/vulcanize/ens_transformers/tree/working)
#### composeAndExecute configuration
A .toml config file is specified when executing the command:
@ -312,9 +311,9 @@ The config provides information for composing a set of transformers:
- `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package).
- Transformer repositories need to be cloned into the user's $GOPATH (`go get`)
- `type` is the type of the transformer; indicating which type of watcher it works with (for now, there are only two options: `eth_event` and `eth_storage`)
- `eth_storage` indicates the transformer works with the [storage watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go)
- `eth_storage` indicates the transformer works with the [storage watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/storage_watcher.go)
that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS)
- `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go)
- `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go)
that fetches event logs from an ETH node
- `migrations` is the relative path from `repository` to the db migrations directory for the transformer
- Note: If any of the imported transformers need additional config variables those need to be included as well
@ -355,14 +354,16 @@ func (e exporter) Export() []interface1.TransformerInitializer, []interface1.Sto
#### Preparing transformer(s) to work as pluggins for composeAndExecute
To plug in an external transformer we need to:
* Create a [package](https://github.com/vulcanize/mcd_transformers/blob/staging/transformers/bite/initializer/initializer.go)
that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33)
or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), respectively
* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83)
or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53) watchers
* Create a [package](https://github.com/vulcanize/ens_transformers/blob/working/transformers/registry/new_owner/initializer/initializer.go)
that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/event_transformer.go#L33)
or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/storage_transformer.go#L31), respectively
* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go#L83)
or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/storage_watcher.go#L58) watcher execution modes
* Create db migrations to run against vulcanizeDB so that we can store the transformer output
* Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
* Do not `goose fix` the transformer migrations
* Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
* If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
as they are [here](https://github.com/vulcanize/vulcanizedb/tree/master/db/migrations)
To update a plugin repository with changes to the core vulcanizedb repository, replace the vulcanizedb vendored in the plugin repo (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb`)
with the newly updated version

64
Supfile
View File

@ -1,64 +0,0 @@
---
version: 0.5
env:
VDB_PATH: /root/go_projects/src/github.com/vulcanize/vulcanizedb
VDB_PG_USER: vulcanize
VDB_PG_PW: vulcanize
networks:
staging:
hosts:
- root@147.75.96.51
prod:
hosts:
- root@147.75.197.13
targets:
deploy:
- remove
- transfer
- buildPostgraphile
- buildVDB
- migrate
- lightSync
- postgraphile
commands:
remove:
desc: remove old vulcanizedb
run: rm -rf $VDB_PATH && rm -rf /usr/local/vulcanizedb && mkdir -p $VDB_PATH
transfer:
desc: transfer repo to remote server
upload:
- src: .
dst: $VDB_PATH
migrate:
desc: run migration
run: >
cd $VDB_PATH &&
make installtools &&
cd db/migrations &&
/root/go_projects/bin/goose postgres "postgresql://$(VDB_PG_USER):$(VDB_PG_PW)@127.0.0.1:5432/vulcanize_public?sslmode=disable" up
buildPostgraphile:
desc: build postgraphile app
run: >
cd $VDB_PATH/postgraphile &&
yarn && tsc
buildVDB:
desc: build vulcanizedb
run: >
cd $VDB_PATH &&
GOPATH=$HOME/go_projects go get &&
GOPATH=$HOME/go_projects go build &&
cp -r . /usr/local/vulcanizedb
lightSync:
desc: start vdb light sync
run: >
systemctl daemon-reload &&
sudo systemctl restart vulcanizedb_light_sync.service &&
sudo systemctl restart vulcanizedb_log_sync.service &&
sudo systemctl restart vulcanizedb_recheck_sync.service
postgraphile:
desc: start postgraphile
run: systemctl daemon-reload && sudo systemctl restart postgraphile.service

5
bin/deploy.sh
View File

@ -1,5 +0,0 @@
if [ $TRAVIS_BRANCH == 'staging' ]; then
sup --debug staging deploy
elif [ $TRAVIS_BRANCH == 'master' ]; then
sup --debug prod deploy
fi

2
cmd/root.go
View File

@ -94,7 +94,6 @@ func init() {
rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
@ -105,7 +104,6 @@ func init() {
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
}

0
db/migrations/00043_create_headers_table.sql → db/migrations/00042_create_headers_table.sql
View File

14
db/migrations/00042_create_token_supply_table.sql
View File

@ -1,14 +0,0 @@
-- +goose Up
CREATE TABLE token_supply (
id SERIAL,
block_id INTEGER NOT NULL,
supply DECIMAL NOT NULL,
token_address CHARACTER VARYING(66) NOT NULL,
CONSTRAINT blocks_fk FOREIGN KEY (block_id)
REFERENCES blocks (id)
ON DELETE CASCADE
);
-- +goose Down
DROP TABLE token_supply;

0
db/migrations/00044_create_checked_headers_table.sql → db/migrations/00043_create_checked_headers_table.sql
View File

0
db/migrations/00045_create_queued_storage.sql → db/migrations/00044_create_queued_storage.sql
View File

51
db/schema.sql
View File

@ -2,8 +2,8 @@
-- PostgreSQL database dump
--
-- Dumped from database version 10.5
-- Dumped by pg_dump version 10.5
-- Dumped from database version 10.6
-- Dumped by pg_dump version 10.6
SET statement_timeout = 0;
SET lock_timeout = 0;
@ -368,38 +368,6 @@ CREATE SEQUENCE public.receipts_id_seq
ALTER SEQUENCE public.receipts_id_seq OWNED BY public.receipts.id;
--
-- Name: token_supply; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.token_supply (
id integer NOT NULL,
block_id integer NOT NULL,
supply numeric NOT NULL,
token_address character varying(66) NOT NULL
);
--
-- Name: token_supply_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.token_supply_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: token_supply_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.token_supply_id_seq OWNED BY public.token_supply.id;
--
-- Name: transactions; Type: TABLE; Schema: public; Owner: -
--
@ -555,13 +523,6 @@ ALTER TABLE ONLY public.queued_storage ALTER COLUMN id SET DEFAULT nextval('publ
ALTER TABLE ONLY public.receipts ALTER COLUMN id SET DEFAULT nextval('public.receipts_id_seq'::regclass);
--
-- Name: token_supply id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.token_supply ALTER COLUMN id SET DEFAULT nextval('public.token_supply_id_seq'::regclass);
--
-- Name: transactions id; Type: DEFAULT; Schema: public; Owner: -
--
@ -754,14 +715,6 @@ ALTER TABLE ONLY public.receipts
ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: token_supply blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.token_supply
ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: checked_headers checked_headers_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--

275
integration_test/omni_full_transformer_test.go Normal file
View File

@ -0,0 +1,275 @@
package integration
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Omni full transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var blockRepository repositories.BlockRepository
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194633)))
Expect(c.LastBlock).To(Equal(int64(6194634)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
})
Describe("Execute", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.TransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
Expect(log.Block).To(Equal(int64(6194634)))
Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.NewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
Expect(log.Block).To(Equal(int64(6194635)))
Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(3))
b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
})

413
integration_test/omni_light_transformer_test.go Normal file
View File

@ -0,0 +1,413 @@
package integration
import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
)
var _ = Describe("Omnit light transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository
var headerID, headerID2 int64
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
headerRepository = repositories.NewHeaderRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194632)))
Expect(c.LastBlock).To(Equal(int64(-1)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
})
})
Describe("Execute- against TrueUSD contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedAddrs)).To(Equal(4))
Expect(len(c.EmittedHashes)).To(Equal(0))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(2))
Expect(len(c.EmittedAddrs)).To(Equal(0))
b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
Expect(ok).To(Equal(false))
})
It("Polls given method using list of collected hashes", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not persist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against both ENS and TrueUSD", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
header4, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header5, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header6, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
headerRepository.CreateOrUpdateHeader(header4)
headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header6)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
newOwnerLog := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
transferLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(transferLog.HeaderID).To(Equal(headerID))
Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(transferLog.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
ens, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
tusd, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(ens.EmittedHashes)).To(Equal(2))
Expect(len(ens.EmittedAddrs)).To(Equal(0))
Expect(len(tusd.EmittedAddrs)).To(Equal(4))
Expect(len(tusd.EmittedHashes)).To(Equal(0))
b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
It("Polls given methods for each contract, using list of collected values", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
owner := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).To(HaveOccurred())
bal := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).ToNot(HaveOccurred())
Expect(bal.Balance).To(Equal("55849938025000000000000"))
Expect(bal.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).To(HaveOccurred())
})
})
})

8
libraries/shared/README.md Normal file
View File

@ -0,0 +1,8 @@
# Shared Tools
## Description
Code that is useful for or used by plugins written on top of VulcanizeDB.
## Note
Much code in this directory may not be used outside of the tests, but don't delete it - it could be used by a plugin.
Renaming and/or deleting functions in this namespace requires a version bump to avoid breaking plugins.

6
libraries/shared/chunker/log_chunker_test.go
View File

@ -58,9 +58,9 @@ var _ = Describe("Log chunker", func() {
Describe("initialisation", func() {
It("creates lookup maps correctly", func() {
Expect(chunker.AddressToNames).To(Equal(map[string][]string{
"0x00000000000000000000000000000000000000a1": []string{"TransformerA"},
"0x00000000000000000000000000000000000000a2": []string{"TransformerA", "TransformerC"},
"0x00000000000000000000000000000000000000b1": []string{"TransformerB"},
"0x00000000000000000000000000000000000000a1": {"TransformerA"},
"0x00000000000000000000000000000000000000a2": {"TransformerA", "TransformerC"},
"0x00000000000000000000000000000000000000b1": {"TransformerB"},
}))
Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{

3
libraries/shared/constants/data.go
View File

@ -17,6 +17,3 @@
package constants
var DataItemLength = 32
// TODO Grab this from DB, since it can change through governance
var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours

388
libraries/shared/factories/README.md Normal file
View File

@ -0,0 +1,388 @@
# Watching Contract Events
One approach VulcanizeDB takes to caching and indexing smart contracts is to watch contract events emitted in receipt logs.
With a light synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
fetch and verify relevant event logs from a full Ethereum node, keeping track of which headers we have checked for which events
with our `checked_headers` table.
## Assumptions
This approach assumes you are running a vDB light sync which is itself run against a light Ethereum node,
this approach also assumes there is a full node available.
Looking forward, we will be building fetchers that enable sourcing data from IPFS instead of an ETH node.
## Shared Code
VulcanizeDB has shared code built out for building and plugging in event transformers
### [Event Watcher (light sync)](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go)
The event watcher is responsible for continuously fetching and delegating chunks of logs and their associated header to the appropriate transformers.
Using the `compose` or `composeAndExecute` command, event watchers can be loaded with plugin event transformers and execute over them.
### [Event Transformer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/event_transformer.go)
The event transformer is responsible for converting event logs into more useful data objects and storing them in Postgres.
The event transformer is composed of converter and repository interfaces and a config struct:
```go
type Transformer struct {
Config transformer.TransformerConfig
Converter Converter
Repository Repository
}
```
The event transformer executes over provided event logs at a given header.
In this process, the converter unpacks these logs into entities and then converts these entities
to their final db models. These models are then written to the Postgres db by the repository.
```go
func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
transformerName := transformer.Config.TransformerName
config := transformer.Config
if len(logs) < 1 {
err := transformer.Repository.MarkHeaderChecked(header.Id)
if err != nil {
log.Printf("Error marking header as checked in %v: %v", transformerName, err)
return err
}
return nil
}
entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
if err != nil {
log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
return err
}
models, err := transformer.Converter.ToModels(entities)
if err != nil {
log.Printf("Error converting entities to models in %v: %v", transformerName, err)
return err
}
err = transformer.Repository.Create(header.Id, models)
if err != nil {
log.Printf("Error persisting %v record: %v", transformerName, err)
return err
}
return nil
}
```
## Custom Code
In order to watch events at a smart contract, for those events the developer must create:
1. Config - struct to hold configuration information (contract address, starting block, event name and signature).
1. Entity - struct to unpack the event log into.
1. Model - struct representing the final data model we want to write to Postgres.
1. Converter - an interface which can unpack event logs into our entities and convert those entities to our models.
1. Repository - an interface to write our models to Postgres.
1. TransformerInitializer - a public variable which exports our configured transformer to be loaded as part of a plugin.
1. DB migrations - migrations to generate the Postgres schema, tables, views, function, etc that are needed to store and interface with the transformed data models.
The example event we will use looks like:
```
event ExampleEvent(bytes32 indexed arg1, address indexed arg2, bytes32 arg3, uint256 arg4, uint256 arg5);
```
### Config
The config holds configuration variables for the event transformer, including a name for the transformer, the contract address
it is working at, the contract's ABI, the topic (e.g. event signature; topic0) that it is filtering for, and starting
and ending block numbers.
```go
type TransformerConfig struct {
TransformerName string
ContractAddresses []string
ContractAbi string
Topic string
StartingBlockNumber int64
EndingBlockNumber int64 // Set -1 for indefinite transformer
}
```
### Entity
Entity field names for event arguments need to be exported and match the argument's name and type. LogIndex,
TransactionIndex, and the Raw log are retained in order to link the data to it's source for downstream validation.
```go
type ExampleEntity struct {
Arg1 common.Hash
Arg2 common.Address
Arg3 common.Hash
Arg4 *big.Int
Arg5 *big.Int
LogIndex uint
TransactionIndex uint
Raw types.Log
}
```
### Model
Model fields are not constrained by the event log structure.
This allows us to rename our fields, decode or convert our log values into more useful types, and perform operations
with or on the values before persisting the data to Postgres.
```go
type ExampleModel struct {
EventHash string
UserAddress string
FractionSkimmed string
Surplus string
Deficit string
FinalPosition string
LogIndex uint
TransactionIndex uint
Raw types.Log
}
```
### Converter
The converter needs to satisfy the interface. One for unpacking logs into the custom defined entities, and
another for converting those entities to their final db models.
```go
type Converter interface {
ToEntities(contractAbi string, ethLog []types.Log) ([]interface{}, error)
ToModels([]interface{}) ([]interface{}, error)
}
```
For the example event, this might look like:
```go
type ExampleConverter struct{}
func (ExampleConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
var entities []interface{}
for _, ethLog := range ethLogs {
entity := &ExampleEntity{}
address := ethLog.Address
abi, err := geth.ParseAbi(contractAbi)
if err != nil {
return nil, err
}
contract := bind.NewBoundContract(address, abi, nil, nil, nil)
err = contract.UnpackLog(entity, "ExampleEvent", ethLog)
if err != nil {
return nil, err
}
entity.Raw = ethLog
entity.LogIndex = ethLog.Index
entity.TransactionIndex = ethLog.TxIndex
entities = append(entities, *entity)
}
return entities, nil
}
func (converter ExampleConverter) ToModels(entities []interface{}) ([]interface{}, error) {
var models []interface{}
for _, entity := range entities {
entity, ok := entity.(ExampleModel)
if !ok {
return nil, fmt.Errorf("entity of type %T, not %T", entity, ExampleModel{})
}
fractionSkimmed, err := hexutil.DecodeBig(entity.Arg3.Hex())
if err != nil {
reuturn nil, err
}
position := new(big.Int)
position.Sub(entity.Arg4, entity.Arg5)
finalPosition := new(big.Int)
if preTaxPosition.Sign() < 0 {
finalPosition = position
} else {
skim := new(big.Int)
skim.Div(position, fractionSkimmed)
finalPosition = position.Sub(position, skim)
}
rawLog, err := json.Marshal(entity.Raw)
if err != nil {
return nil, err
}
model := ExampleModel{
EventHash: entity.Arg1.Hex(),
UserAddress: entity.Arg2.Hex(),
FractionSkimmed: fractionSkimmed.String(),
Surplus: entity.Arg4.String(),
Deficit: entity.Arg5.String(),
FinalPosition: finalPosition,
LogIndex: entity.LogIndex,
TransactionIndex: entity.TransactionIndex,
Raw: rawLog,
}
models = append(models, model)
}
return models, nil
}
```
Notice that in this example we have a bytes32 argument in the event that needs to be decoded to an integer before it can be worked with
to produce our hypothetical `FinalPosition` field. This is to highlight the fact that contracts can and sometimes do encode the
data types we want to work with into raw bytes. Writing custom transformers with these converters allows us to account for this.
### Repository
The repository needs to satisfy the interface and use the `Create` method to write the model to Postgres.
```go
type Repository interface {
Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
SetDB(db *postgres.DB)
}
```
For the example event, this might look like:
```go
type ExampleRepository struct {
db *postgres.DB
}
func (repository *ExampleRepository) SetDB(db *postgres.DB) {
repository.db = db
}
func (repository ExampleRepository) Create(headerID int64, models []interface{}) error {
tx, dBaseErr := repository.db.Begin()
if dBaseErr != nil {
return dBaseErr
}
for _, model := range models {
model, ok := model.(ExampleModel)
if !ok {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return fmt.Errorf("model of type %T, not %T", model, ExampleModel{})
}
_, execErr := tx.Exec(
`INSERT into example_schema.example_event (header_id, event_hash, user_address, fraction_skimmed, surplus, deficit, final_position, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET event_hash = $2, user_address = $3, fraction_skimmed = $4, surplus = $5, deficit = $6, final_position = $7, raw_log = $10;`,
headerID, model.EventHash, model.UserAddress, model.FractonSkimmed, model.Surplus, model.Deficit, model.FinalPosition, model.LogIndex, model.TransactionIndex, model.Raw,
)
if execErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return execErr
}
}
checkHeaderErr := repo.MarkHeaderCheckedInTransaction(headerID, tx, "example_event_checked")
if checkHeaderErr != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
log.Error("failed to rollback ", rollbackErr)
}
return checkHeaderErr
}
return tx.Commit()
}
func (repository ExampleRepository) MarkHeaderChecked(headerID int64) error {
return repo.MarkHeaderChecked(headerID, repository.db, "example_event_checked")
}
func (repository ExampleRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
return repo.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db,"example_event_checked")
}
func (repository ExampleRepository) RecheckHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
return repo.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, "example_event_checked")
}
```
### TransformerInitializer
A transformer initializer variable needs to be exported from somewhere within the transformer repository so that the transformer can be
loaded as part of a plugin in the `compose` or `composeAndExecute` commands. It is important that this variable is named `TransformerInitializer` and
it must be of `type TransformerInitializer func(db *postgres.DB) EventTransformer`.
```go
var TransformerInitializer transformer.TransformerInitializer = factories.Transformer{
Config: exampleEventConfig,
Converter: ExampleConverter{},
Repository: &ExampleRepository{},
}.NewTransformer
```
### DB migrations
We use `goose` as our migration management tool. Any Go data model that needs to be written to Postgres by the
repository needs a db migration for the corresponding Postgres data model.
Each contract or set of transformers being watched should define its own namespace with a db schema:
```postgresql
-- +goose Up
CREATE SCHEMA example_schema;
-- +goose Down
DROP SCHEMA example_schema;
```
For the example event and its resulting model, the table we write to would look like:
```postgresql
-- +goose Up
CREATE TABLE example_schema.example_event (
id SERIAL PRIMARY KEY,
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
event_hash CHARACTER VARYING(66) NOT NULL,
user_address CHARACTER VARYING(66) NOT NULL,
fraction_skimmed NUMERIC NOT NULL,
surplus NUMERIC NOT NULL,
deficit NUMERIC NOT NULL,
final_position NUMERIC NOT NULL,
tx_idx INTEGER NOT NUll,
log_idx INTEGER NOT NUll,
raw_log JSONB,
UNIQUE (header_id, tx_idx, log_idx)
);
ALTER TABLE public.checked_headers
ADD COLUMN example_event_checked INTEGER NOT NULL DEFAULT 0;
-- +goose Down
DROP TABLE maker.bite;
ALTER TABLE public.checked_headers
DROP COLUMN bite_checked;
```
Notice that we have also added a column to the `checked_headers` table for this event so that we can keep track
of which headers we have already filtered through for this event.
## Summary
To create a transformer for a contract event we need to create entities for unpacking the raw log, models to represent
the final data structure, a converter to mediate this unpacking and conversion between entities to models, a repository to write
these models to Postgres, db migrations to accommodate these models in Postgres, and a TransformerInitializer to export the
configured transformer and load it as a plugin to the `compose` or `composeAndExecute` commands as described in the [main readme](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/README.md#composeandexecute-configuration).

10
libraries/shared/factories/storage/EXAMPLE.md
View File

@ -26,7 +26,7 @@ contract Contract {
function add_address(address addr) public {
bool exists = addresses[addr] > 0;
addresses[addr] = addresses[addr] + 1;
addresses[addr]++;
if (!exists) {
emit AddressAdded(addr, ++num_addresses);
}
@ -36,7 +36,7 @@ contract Contract {
Disclaimer: this contract has not been audited and is not intended to be modeled or used in production. :)
This contract persists two values in it's storage:
This contract persists two values in its storage:
1. `num_addresses`: the total number of unique addresses known to the contract.
2. `addresses`: a mapping that records the number of times an address has been added to the contract.
@ -131,9 +131,9 @@ Once we have recognized a storage diff, we can decode the storage value to the d
Since the metadata tells us that the above values are `uint256`, we can decode a value like `0000000000000000000000000000000000000000000000000000000000000001` to `1`.
The purpose of the contract-specific repository is to write that value to the database in a way that makes it useful for future queries.
Typically, the involves writing the block hash, block number, decoded value, and any keys in the metadata to a table.
Typically, this involves writing the block hash, block number, decoded value, and any keys in the metadata to a table.
The current repository interface has a generalized `Create` function that can accept any arbitrary storage row along with it's metadata.
The current repository interface has a generalized `Create` function that can accept any arbitrary storage row along with its metadata.
This is deliberate, to facilitate shared use of the common storage transformer.
An implication of this decision is that the `Create` function typically includes a `switch` statement that selects which table to write to, as well as what data to include, based on the name of the variable as defined in the metadata.
@ -158,7 +158,7 @@ func (repository AddressStorageRepository) Create(blockNumber int, blockHash str
## Summary
With our very simple address storing contract, we would be able to read it's storage diffs by implementing an event transformer, a mappings, and a repository.
With our very simple address storing contract, we would be able to read its storage diffs by implementing an event transformer, a mappings, and a repository.
The mappings would be able to lookup storage keys reflecting `num_addresses` or any slot in `addresses`, using addresses derived from watching the `AddressAdded` event for the latter.

2
libraries/shared/factories/storage/README.md
View File

@ -23,7 +23,7 @@ VulcanizeDB has shared code for continuously reading from the CSV file written b
The storage watcher is responsible for continuously delegating CSV rows to the appropriate transformer as they are being written by the ethereum node.
It maintains a mapping of contract addresses to transformers, and will ignore storage diff rows for contract addresses that do not have a corresponding transformer.
The storage watcher is currently initialized from the `parseStorageDiffs` command, which also adds transformers that the watcher should know about in its mapping of addresses to transformers.
Storage watchers can be loaded with plugin storage transformers and executed using the `composeAndExecute` command.
### Storage Transformer

15
libraries/shared/repository/repository.go
View File

@ -18,9 +18,9 @@ package repository
import (
"bytes"
"database/sql"
"database/sql/driver"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -35,7 +35,7 @@ func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn str
return err
}
func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
@ -155,14 +155,3 @@ func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.Transfor
return result.String()
}
func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) {
var blockTimestamp int64
err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp)
if err != nil {
return 0, err
}
tic := blockTimestamp + constants.TTL
return tic, nil
}

189
libraries/shared/repository/repository_utility_test.go → libraries/shared/repository/repository_test.go
View File

@ -19,6 +19,7 @@ package repository_test
import (
"fmt"
"math/rand"
"strconv"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -34,7 +35,83 @@ import (
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Repository utilities", func() {
var _ = Describe("Repository", func() {
Describe("MarkHeaderChecked", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
It("marks passed column as checked for passed header", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
err := shared.MarkHeaderChecked(headerID, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
})
Describe("MarkHeaderCheckedInTransaction", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
It("marks passed column as checked for passed header within a passed transaction", func() {
headerRepository := repositories.NewHeaderRepository(db)
headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
Expect(headerErr).NotTo(HaveOccurred())
tx, txErr := db.Beginx()
Expect(txErr).NotTo(HaveOccurred())
err := shared.MarkHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).NotTo(HaveOccurred())
var checkedCount int
fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
Expect(fetchErr).NotTo(HaveOccurred())
Expect(checkedCount).To(Equal(1))
})
})
Describe("MissingHeaders", func() {
var (
db *postgres.DB
@ -116,6 +193,84 @@ var _ = Describe("Repository utilities", func() {
})
})
Describe("RecheckHeaders", func() {
var (
checkedHeadersColumn string
db *postgres.DB
headerOneID, headerTwoID, headerThreeID, headerFourID int64
headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
// create header checked column
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
// create headers
headerRepository := repositories.NewHeaderRepository(db)
headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
Expect(headerOneErr).NotTo(HaveOccurred())
headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
Expect(headerTwoErr).NotTo(HaveOccurred())
headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
Expect(headerThreeErr).NotTo(HaveOccurred())
headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
Expect(headerFourErr).NotTo(HaveOccurred())
// mark every header checked at least once, with one fully rechecked (headerThree)
maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
Expect(intConversionErr).NotTo(HaveOccurred())
_, markHeaderOneCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerOneID, maxCheckCount)
Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
_, markHeaderTwoCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerTwoID, maxCheckCount)
Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
_, markHeaderThreeCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerThreeID, maxCheckCount+1)
Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
_, markHeaderFourCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerFourID, maxCheckCount)
Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
Describe("when no ending block number (ending block number == -1)", func() {
It("returns all headers since starting block where checked count is less than cap", func() {
headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
})
})
Describe("when ending block number specified", func() {
It("returns headers between starting and ending block where checked count is less than cap", func() {
headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
})
})
})
Describe("GetCheckedColumnNames", func() {
It("gets the column names from checked_headers", func() {
db := test_config.NewTestDB(test_config.NewTestNode())
@ -162,33 +317,9 @@ var _ = Describe("Repository utilities", func() {
func getExpectedColumnNames() []string {
return []string{
"price_feeds_checked",
"flip_kick_checked",
"frob_checked",
"tend_checked",
"bite_checked",
"dent_checked",
"pit_file_debt_ceiling_checked",
"pit_file_ilk_checked",
"vat_init_checked",
"drip_file_ilk_checked",
"drip_file_repo_checked",
"drip_file_vow_checked",
"deal_checked",
"drip_drip_checked",
"cat_file_chop_lump_checked",
"cat_file_flip_checked",
"cat_file_pit_vow_checked",
"flop_kick_checked",
"vat_move_checked",
"vat_fold_checked",
"vat_heal_checked",
"vat_toll_checked",
"vat_tune_checked",
"vat_grab_checked",
"vat_flux_checked",
"vat_slip_checked",
"vow_flog_checked",
"flap_kick_checked",
"column_1_checked",
"column_2_checked",
"column_3_checked",
"column_4_checked",
}
}

65
libraries/shared/storage/mappings_test.go Normal file
View File

@ -0,0 +1,65 @@
package storage_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
)
var _ = Describe("Mappings", func() {
Describe("GetMapping", func() {
It("returns the storage key for a mapping when passed the mapping's index on the contract and the desired value's key", func() {
// ex. solidity:
// mapping (bytes32 => uint) public amounts
// to access amounts, pass in the index of the mapping on the contract + the bytes32 key for the uint val being looked up
indexOfMappingOnContract := storage.IndexZero
keyForDesiredValueInMapping := "1234567890abcdef"
storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
Describe("GetNestedMapping", func() {
It("returns the storage key for a nested mapping when passed the mapping's index on the contract and the desired value's keys", func() {
// ex. solidity:
// mapping (bytes32 => uint) public amounts
// mapping (address => mapping (uint => bytes32)) public addressNames
// to access addressNames, pass in the index of the mapping on the contract + the address and uint keys for the bytes32 val being looked up
indexOfMappingOnContract := storage.IndexOne
keyForOuterMapping := "1234567890abcdef"
keyForInnerMapping := "123"
storageKey := storage.GetNestedMapping(indexOfMappingOnContract, keyForOuterMapping, keyForInnerMapping)
expectedStorageKey := common.HexToHash("0x82113529f6cd61061d1a6f0de53f2bdd067a1addd3d2b46be50a99abfcdb1661")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
Describe("GetIncrementedKey", func() {
It("returns the storage key for later values sharing an index on the contract with other earlier values", func() {
// ex. solidity:
// mapping (bytes32 => uint) public amounts
// mapping (address => mapping (uint => bytes32)) public addressNames
// struct Data {
// uint256 quantity;
// uint256 quality;
// }
// mapping (bytes32 => Data) public itemData;
// to access quality from itemData, pass in the storage key for the zero-indexed value (quantity) + the number of increments required.
// (For "quality", we must increment the storage key for the corresponding "quantity" by 1).
indexOfMappingOnContract := storage.IndexTwo
keyForDesiredValueInMapping := "1234567890abcdef"
storageKeyForFirstPropertyOnStruct := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
storageKey := storage.GetIncrementedKey(storageKeyForFirstPropertyOnStruct, 1)
expectedStorageKey := common.HexToHash("0x69b38749f0a8ed5d505c8474f7fb62c7828aad8a7627f1c67e07af1d2368cad4")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
})

7
libraries/shared/storage/utils/decoder.go
View File

@ -27,6 +27,8 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err
switch metadata.Type {
case Uint256:
return decodeUint256(row.StorageValue.Bytes()), nil
case Uint48:
return decodeUint48(row.StorageValue.Bytes()), nil
case Address:
return decodeAddress(row.StorageValue.Bytes()), nil
case Bytes32:
@ -41,6 +43,11 @@ func decodeUint256(raw []byte) string {
return n.String()
}
func decodeUint48(raw []byte) string {
n := big.NewInt(0).SetBytes(raw)
return n.String()
}
func decodeAddress(raw []byte) string {
return common.BytesToAddress(raw).Hex()
}

11
libraries/shared/storage/utils/decoder_test.go
View File

@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
})
It("decodes uint48", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
row := utils.StorageDiffRow{StorageValue: fakeInt}
metadata := utils.StorageValueMetadata{Type: utils.Uint48}
result, err := utils.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
})
It("decodes address", func() {
fakeAddress := common.HexToAddress("0x12345")
row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()}

1
libraries/shared/storage/utils/value.go
View File

@ -20,6 +20,7 @@ type ValueType int
const (
Uint256 ValueType = iota
Uint48
Bytes32
Address
)

22
libraries/shared/storage/utils/value_test.go Normal file
View File

@ -0,0 +1,22 @@
package utils_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
)
var _ = Describe("Storage value metadata getter", func() {
It("returns a storage value metadata instance with corresponding fields assigned", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.Uint256
expectedMetadata := utils.StorageValueMetadata{
Name: metadataName,
Keys: metadataKeys,
Type: metadataType,
}
Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
})
})

2
libraries/shared/test_data/generic.go
View File

@ -49,7 +49,7 @@ var GenericTestConfig = transformer.TransformerConfig{
}
func randomString(length int) string {
var seededRand *rand.Rand = rand.New(
var seededRand = rand.New(
rand.NewSource(time.Now().UnixNano()))
charset := "abcdefghijklmnopqrstuvwxyz1234567890"
b := make([]byte, length)

8
libraries/shared/watcher/storage_watcher.go
View File

@ -50,8 +50,8 @@ func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
for _, initializer := range initializers {
transformer := initializer(watcher.db)
watcher.Transformers[transformer.ContractAddress()] = transformer
storageTransformer := initializer(watcher.db)
watcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
}
}
@ -65,12 +65,12 @@ func (watcher StorageWatcher) Execute() error {
if parseErr != nil {
return parseErr
}
transformer, ok := watcher.Transformers[row.Contract]
storageTransformer, ok := watcher.Transformers[row.Contract]
if !ok {
logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
continue
}
executeErr := transformer.Execute(row)
executeErr := storageTransformer.Execute(row)
if executeErr != nil {
if isKeyNotFound(executeErr) {
queueErr := watcher.Queue.Add(row)

6
main.go
View File

@ -4,8 +4,6 @@ import (
"github.com/vulcanize/vulcanizedb/cmd"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
"os"
)
@ -19,9 +17,5 @@ func main() {
log.Info("Failed to log to file, using default stderr")
}
tracer.Start(tracer.WithServiceName(viper.GetString("datadog.name")))
cmd.Execute()
defer tracer.Stop()
}

11
pkg/datastore/postgres/repositories/block_repository.go
View File

@ -17,7 +17,6 @@
package repositories
import (
"context"
"database/sql"
"errors"
log "github.com/sirupsen/logrus"
@ -122,7 +121,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) {
var blockId int64
tx, _ := blockRepository.database.BeginTx(context.Background(), nil)
tx, _ := blockRepository.database.Beginx()
err := tx.QueryRow(
`INSERT INTO blocks
(eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
@ -145,7 +144,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
return blockId, nil
}
func (blockRepository BlockRepository) createTransactions(tx *sql.Tx, blockId int64, transactions []core.Transaction) error {
func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockId int64, transactions []core.Transaction) error {
for _, transaction := range transactions {
err := blockRepository.createTransaction(tx, blockId, transaction)
if err != nil {
@ -165,7 +164,7 @@ func nullStringToZero(s string) string {
return s
}
func (blockRepository BlockRepository) createTransaction(tx *sql.Tx, blockId int64, transaction core.Transaction) error {
func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.Transaction) error {
_, err := tx.Exec(
`INSERT INTO transactions
(block_id, hash, nonce, tx_to, tx_from, gaslimit, gasprice, value, input_data)
@ -198,7 +197,7 @@ func hasReceipt(transaction core.Transaction) bool {
return transaction.Receipt.TxHash != ""
}
func (blockRepository BlockRepository) createReceipt(tx *sql.Tx, blockId int64, receipt core.Receipt) (int, error) {
func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) {
//Not currently persisting log bloom filters
var receiptId int
err := tx.QueryRow(
@ -224,7 +223,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
return retrievedBlockHash, blockExists(retrievedBlockHash)
}
func (blockRepository BlockRepository) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error {
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error {
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

3
pkg/datastore/postgres/repositories/logs_repository.go
View File

@ -17,7 +17,6 @@
package repositories
import (
"context"
"github.com/sirupsen/logrus"
"database/sql"
@ -31,7 +30,7 @@ type LogRepository struct {
}
func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error {
tx, _ := logRepository.DB.BeginTx(context.Background(), nil)
tx, _ := logRepository.DB.Beginx()
for _, tlog := range lgs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

10
pkg/datastore/postgres/repositories/receipt_repository.go
View File

@ -17,8 +17,8 @@
package repositories
import (
"context"
"database/sql"
"github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
@ -31,7 +31,7 @@ type ReceiptRepository struct {
}
func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
tx, err := receiptRepository.DB.BeginTx(context.Background(), nil)
tx, err := receiptRepository.DB.Beginx()
if err != nil {
return err
}
@ -53,7 +53,7 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64,
return nil
}
func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, error) {
func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
var receiptId int64
err := tx.QueryRow(
`INSERT INTO receipts
@ -68,7 +68,7 @@ func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, erro
return receiptId, err
}
func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
for _, log := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@ -84,7 +84,7 @@ func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
}
func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
tx, _ := receiptRepository.DB.BeginTx(context.Background(), nil)
tx, _ := receiptRepository.DB.Beginx()
var receiptId int64
err := tx.QueryRow(
`INSERT INTO receipts

3
pkg/fakes/data.go
View File

@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
@ -49,5 +48,3 @@ func GetFakeHeader(blockNumber int64) core.Header {
Timestamp: strconv.FormatInt(fakeTimestamp, 10),
}
}
var FakeHeaderTic = fakeTimestamp + constants.TTL

14
pkg/fakes/mock_filter_repository.go Normal file
View File

@ -0,0 +1,14 @@
package fakes
import "github.com/vulcanize/vulcanizedb/pkg/filters"
type MockFilterRepository struct {
}
func (*MockFilterRepository) CreateFilter(filter filters.LogFilter) error {
return nil
}
func (*MockFilterRepository) GetFilter(name string) (filters.LogFilter, error) {
panic("implement me")
}

15
pkg/fakes/mock_full_block_retriever.go Normal file
View File

@ -0,0 +1,15 @@
package fakes
type MockFullBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
MostRecentBlock int64
}
func (retriever *MockFullBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockFullBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return retriever.MostRecentBlock, nil
}

14
pkg/fakes/mock_light_block_retriever.go Normal file
View File

@ -0,0 +1,14 @@
package fakes
type MockLightBlockRetriever struct {
FirstBlock int64
FirstBlockErr error
}
func (retriever *MockLightBlockRetriever) RetrieveFirstBlock() (int64, error) {
return retriever.FirstBlock, retriever.FirstBlockErr
}
func (retriever *MockLightBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
return 0, nil
}

42
pkg/fakes/mock_light_header_repository.go Normal file
View File

@ -0,0 +1,42 @@
package fakes
import "github.com/vulcanize/vulcanizedb/pkg/core"
type MockLightHeaderRepository struct {
}
func (*MockLightHeaderRepository) AddCheckColumn(id string) error {
return nil
}
func (*MockLightHeaderRepository) AddCheckColumns(ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
panic("implement me")
}
func (*MockLightHeaderRepository) CheckCache(key string) (interface{}, bool) {
panic("implement me")
}

40
pkg/fakes/mock_parser.go Normal file
View File

@ -0,0 +1,40 @@
package fakes
import (
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
type MockParser struct {
AbiToReturn string
EventName string
Event types.Event
}
func (*MockParser) Parse(contractAddr string) error {
return nil
}
func (*MockParser) ParseAbiStr(abiStr string) error {
panic("implement me")
}
func (parser *MockParser) Abi() string {
return parser.AbiToReturn
}
func (*MockParser) ParsedAbi() abi.ABI {
return abi.ABI{}
}
func (*MockParser) GetMethods(wanted []string) []types.Method {
panic("implement me")
}
func (*MockParser) GetSelectMethods(wanted []string) []types.Method {
return []types.Method{}
}
func (parser *MockParser) GetEvents(wanted []string) map[string]types.Event {
return map[string]types.Event{parser.EventName: parser.Event}
}

24
pkg/fakes/mock_poller.go Normal file
View File

@ -0,0 +1,24 @@
package fakes
import (
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
)
type MockPoller struct {
ContractName string
}
func (*MockPoller) PollContract(con contract.Contract) error {
panic("implement me")
}
func (*MockPoller) PollContractAt(con contract.Contract, blockNumber int64) error {
panic("implement me")
}
func (poller *MockPoller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
if p, ok := result.(*string); ok {
*p = poller.ContractName
}
return nil
}

4
pkg/omni/full/converter/converter.go
View File

@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) {
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
values := make(map[string]interface{})
log := helpers.ConvertToLog(watchedEvent)
err := contract.UnpackLogIntoMap(values, event.Name, log)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}

94
pkg/omni/full/transformer/transformer.go
View File

@ -34,7 +34,7 @@ import (
)
// Requires a fully synced vDB and a running eth node (or infura)
type transformer struct {
type Transformer struct {
// Database interfaces
datastore.FilterRepository // Log filters repo; accepts filters generated by Contract.GenerateFilters()
datastore.WatchedEventRepository // Watched event log views, created by the log filters
@ -76,8 +76,8 @@ type transformer struct {
}
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transformer {
return &transformer{
func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer {
return &Transformer{
Poller: poller.NewPoller(BC, DB, types.FullSync),
Parser: parser.NewParser(network),
BlockRetriever: retriever.NewBlockRetriever(DB),
@ -101,59 +101,59 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (t *transformer) Init() error {
for contractAddr, subset := range t.WatchedEvents {
func (transformer *Transformer) Init() error {
for contractAddr, subset := range transformer.WatchedEvents {
// Get Abi
err := t.Parser.Parse(contractAddr)
err := transformer.Parser.Parse(contractAddr)
if err != nil {
return err
}
// Get first block and most recent block number in the header repo
firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr)
if err != nil {
return err
}
lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock()
if err != nil {
return err
}
// Set to specified range if it falls within the bounds
if firstBlock < t.ContractStart[contractAddr] {
firstBlock = t.ContractStart[contractAddr]
if firstBlock < transformer.ContractStart[contractAddr] {
firstBlock = transformer.ContractStart[contractAddr]
}
// Get contract name if it has one
var name = new(string)
t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock)
transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
for _, arg := range t.EventArgs[contractAddr] {
for _, arg := range transformer.EventArgs[contractAddr] {
eventArgs[arg] = true
}
methodArgs := map[string]bool{}
for _, arg := range t.MethodArgs[contractAddr] {
for _, arg := range transformer.MethodArgs[contractAddr] {
methodArgs[arg] = true
}
// Aggregate info into contract object
info := contract.Contract{
Name: *name,
Network: t.Network,
Network: transformer.Network,
Address: contractAddr,
Abi: t.Parser.Abi(),
ParsedAbi: t.Parser.ParsedAbi(),
Abi: transformer.Parser.Abi(),
ParsedAbi: transformer.Parser.ParsedAbi(),
StartingBlock: firstBlock,
LastBlock: lastBlock,
Events: t.Parser.GetEvents(subset),
Methods: t.Parser.GetSelectMethods(t.WantedMethods[contractAddr]),
Events: transformer.Parser.GetEvents(subset),
Methods: transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]),
FilterArgs: eventArgs,
MethodArgs: methodArgs,
CreateAddrList: t.CreateAddrList[contractAddr],
CreateHashList: t.CreateHashList[contractAddr],
Piping: t.Piping[contractAddr],
CreateAddrList: transformer.CreateAddrList[contractAddr],
CreateHashList: transformer.CreateHashList[contractAddr],
Piping: transformer.Piping[contractAddr],
}.Init()
// Use info to create filters
@ -164,14 +164,14 @@ func (t *transformer) Init() error {
// Iterate over filters and push them to the repo using filter repository interface
for _, filter := range info.Filters {
err = t.CreateFilter(filter)
err = transformer.FilterRepository.CreateFilter(filter)
if err != nil {
return err
}
}
// Store contract info for further processing
t.Contracts[contractAddr] = info
transformer.Contracts[contractAddr] = info
}
return nil
@ -182,18 +182,18 @@ func (t *transformer) Init() error {
// Uses converter to convert logs into custom log type
// Persists converted logs into custuom postgres tables
// Calls selected methods, using token holder address generated during event log conversion
func (tr transformer) Execute() error {
if len(tr.Contracts) == 0 {
func (transformer Transformer) Execute() error {
if len(transformer.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts to work with")
}
// Iterate through all internal contracts
for _, con := range tr.Contracts {
for _, con := range transformer.Contracts {
// Update converter with current contract
tr.Update(con)
transformer.Update(con)
// Iterate through contract filters and get watched event logs
for eventSig, filter := range con.Filters {
watchedEvents, err := tr.GetWatchedEvents(filter.Name)
watchedEvents, err := transformer.GetWatchedEvents(filter.Name)
if err != nil {
return err
}
@ -201,7 +201,7 @@ func (tr transformer) Execute() error {
// Iterate over watched event logs
for _, we := range watchedEvents {
// Convert them to our custom log type
cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig])
if err != nil {
return err
}
@ -211,7 +211,7 @@ func (tr transformer) Execute() error {
// If log is not empty, immediately persist in repo
// Run this in seperate goroutine?
err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
if err != nil {
return err
}
@ -222,7 +222,7 @@ func (tr transformer) Execute() error {
// poller polls select contract methods
// and persists the results into custom pg tables
// Run this in seperate goroutine?
if err := tr.PollContract(*con); err != nil {
if err := transformer.PollContract(*con); err != nil {
return err
}
}
@ -231,41 +231,41 @@ func (tr transformer) Execute() error {
}
// Used to set which contract addresses and which of their events to watch
func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) {
transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to watch events for
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) {
transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
tr.ContractStart[strings.ToLower(contractAddr)] = start
func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) {
transformer.ContractStart[strings.ToLower(contractAddr)] = start
}
// Used to set whether or not to persist an account address list
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[strings.ToLower(contractAddr)] = on
func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) {
transformer.CreateAddrList[strings.ToLower(contractAddr)] = on
}
// Used to set whether or not to persist an hash list
func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
tr.CreateHashList[strings.ToLower(contractAddr)] = on
func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) {
transformer.CreateHashList[strings.ToLower(contractAddr)] = on
}
// Used to turn method piping on for a contract
func (tr *transformer) SetPiping(contractAddr string, on bool) {
tr.Piping[strings.ToLower(contractAddr)] = on
func (transformer *Transformer) SetPiping(contractAddr string, on bool) {
transformer.Piping[strings.ToLower(contractAddr)] = on
}

365
pkg/omni/full/transformer/transformer_test.go
View File

@ -17,353 +17,160 @@
package transformer_test
import (
"fmt"
"math/rand"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
)
var _ = Describe("Transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var blockRepository repositories.BlockRepository
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
var fakeAddress = "0x1234567890abcdef"
rand.Seed(time.Now().UnixNano())
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
blockRepository = *repositories.NewBlockRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
Describe("SetEvents", func() {
It("Sets which events to watch from the given contract address", func() {
watchedEvents := []string{"Transfer", "Mint"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, watchedEvents)
Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, watchedEvents)
Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
})
})
Describe("SetEventAddrs", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEventArgs(fakeAddress, eventAddrs)
Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
})
})
Describe("SetMethods", func() {
It("Sets which methods to poll at the given contract address", func() {
watchedMethods := []string{"balanceOf", "totalSupply"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethods(constants.TusdContractAddress, watchedMethods)
Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethods(fakeAddress, watchedMethods)
Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
})
})
Describe("SetMethodAddrs", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethodArgs(fakeAddress, methodAddrs)
Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
})
})
Describe("SetStartingBlock", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetStartingBlock(constants.TusdContractAddress, 11)
Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetStartingBlock(fakeAddress, 11)
Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
})
})
Describe("SetCreateAddrList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateAddrList(constants.TusdContractAddress, true)
Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateAddrList(fakeAddress, true)
Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
})
})
Describe("SetCreateHashList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateHashList(constants.TusdContractAddress, true)
Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateHashList(fakeAddress, true)
Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
})
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockFullBlockRetriever{}
firstBlock := int64(1)
mostRecentBlock := int64(2)
blockRetriever.FirstBlock = firstBlock
blockRetriever.MostRecentBlock = mostRecentBlock
parsr := &fakes.MockParser{}
fakeAbi := "fake_abi"
eventName := "Transfer"
event := types.Event{}
parsr.AbiToReturn = fakeAbi
parsr.EventName = eventName
parsr.Event = event
pollr := &fakes.MockPoller{}
fakeContractName := "fake_contract_name"
pollr.ContractName = fakeContractName
t := getTransformer(blockRetriever, parsr, pollr)
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
c, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194633)))
Expect(c.LastBlock).To(Equal(int64(6194634)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
Expect(c.StartingBlock).To(Equal(firstBlock))
Expect(c.LastBlock).To(Equal(mostRecentBlock))
Expect(c.Abi).To(Equal(fakeAbi))
Expect(c.Name).To(Equal(fakeContractName))
Expect(c.Address).To(Equal(fakeAddress))
})
It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockFullBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("Does nothing if watched events are unset", func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
t := transformer.NewTransformer("", blockChain, db)
err = t.Init()
t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
_, ok := t.Contracts[tusdAddr]
_, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(false))
})
})
Describe("Execute", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.TransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
Expect(log.Block).To(Equal(int64(6194634)))
Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("0"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.NewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
Expect(log.Block).To(Equal(int64(6194635)))
Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(3))
b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not perist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
})
func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
return transformer.Transformer{
FilterRepository: &fakes.MockFilterRepository{},
Parser: parsr,
BlockRetriever: blockRetriever,
Poller: pollr,
Contracts: map[string]*contract.Contract{},
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
}
}

8
pkg/omni/light/converter/converter.go
View File

@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) {
// Convert the given watched event log into a types.Log for the given event
func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
returnLogs := make([]types.Log, 0, len(logs))
for _, log := range logs {
values := make(map[string]interface{})
@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
values[field.Name] = i
}
err := contract.UnpackLogIntoMap(values, event.Name, log)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}
@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
eventsToLogs := make(map[string][]types.Log)
for _, event := range events {
eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
// If the log is of this event type, process it as such
if event.Sig() == log.Topics[0] {
values := make(map[string]interface{})
err := contract.UnpackLogIntoMap(values, event.Name, log)
err := boundContract.UnpackLogIntoMap(values, event.Name, log)
if err != nil {
return nil, err
}

6
pkg/omni/light/repository/header_repository.go
View File

@ -17,8 +17,8 @@
package repository
import (
"database/sql"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/hashicorp/golang-lru"
@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
}
func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
tx, err := r.db.Begin()
tx, err := r.db.Beginx()
if err != nil {
return err
}
@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
return r.columns.Get(key)
}
func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error {
func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
VALUES ($1, $2)
ON CONFLICT (header_id) DO

2
pkg/omni/light/retriever/retriever_suite_test.go
View File

@ -27,7 +27,7 @@ import (
func TestRetriever(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Light BLock Number Retriever Suite Test")
RunSpecs(t, "Light Block Number Retriever Suite Test")
}
var _ = BeforeSuite(func() {

30
pkg/omni/light/transformer/transformer.go
View File

@ -37,7 +37,7 @@ import (
)
// Requires a light synced vDB (headers) and a running eth node (or infura)
type transformer struct {
type Transformer struct {
// Database interfaces
srep.EventRepository // Holds transformed watched event log data
repository.HeaderRepository // Interface for interaction with header repositories
@ -93,9 +93,9 @@ type transformer struct {
// 4. Execute
// Transformer takes in config for blockchain, database, and network id
func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transformer {
func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer {
return &transformer{
return &Transformer{
Poller: poller.NewPoller(bc, db, types.LightSync),
Fetcher: fetcher.NewFetcher(bc),
Parser: parser.NewParser(network),
@ -120,7 +120,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
// Loops over all of the addr => filter sets
// Uses parser to pull event info from abi
// Use this info to generate event filters
func (tr *transformer) Init() error {
func (tr *Transformer) Init() error {
// Initialize internally configured transformer settings
tr.contractAddresses = make([]string, 0) // Holds all contract addresses, for batch fetching of logs
tr.sortedEventIds = make(map[string][]string) // Map to sort event column ids by contract, for post fetch processing and persisting of logs
@ -154,7 +154,7 @@ func (tr *transformer) Init() error {
// Get contract name if it has one
var name = new(string)
tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock)
// Remove any potential accidental duplicate inputs in arg filter values
eventArgs := map[string]bool{}
@ -221,7 +221,7 @@ func (tr *transformer) Init() error {
return nil
}
func (tr *transformer) Execute() error {
func (tr *Transformer) Execute() error {
if len(tr.Contracts) == 0 {
return errors.New("error: transformer has no initialized contracts")
}
@ -311,7 +311,7 @@ func (tr *transformer) Execute() error {
}
// Used to poll contract methods at a given header
func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
for _, con := range tr.Contracts {
// Skip method polling processes if no methods are specified
// Also don't try to poll methods below this contract's specified starting block
@ -336,41 +336,41 @@ func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[str
}
// Used to set which contract addresses and which of their events to watch
func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) {
tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to watch events for
func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set which contract addresses and which of their methods to call
func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) {
tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
}
// Used to set subset of account addresses to poll methods on
func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
}
// Used to set the block range to watch for a given address
func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) {
tr.ContractStart[strings.ToLower(contractAddr)] = start
}
// Used to set whether or not to persist an account address list
func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) {
tr.CreateAddrList[strings.ToLower(contractAddr)] = on
}
// Used to set whether or not to persist an hash list
func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) {
tr.CreateHashList[strings.ToLower(contractAddr)] = on
}
// Used to turn method piping on for a contract
func (tr *transformer) SetPiping(contractAddr string, on bool) {
func (tr *Transformer) SetPiping(contractAddr string, on bool) {
tr.Piping[strings.ToLower(contractAddr)] = on
}

497
pkg/omni/light/transformer/transformer_test.go
View File

@ -17,491 +17,150 @@
package transformer_test
import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
)
var _ = Describe("Transformer", func() {
var db *postgres.DB
var err error
var blockChain core.BlockChain
var headerRepository repositories.HeaderRepository
var headerID, headerID2 int64
var ensAddr = strings.ToLower(constants.EnsContractAddress)
var tusdAddr = strings.ToLower(constants.TusdContractAddress)
BeforeEach(func() {
db, blockChain = test_helpers.SetupDBandBC()
headerRepository = repositories.NewHeaderRepository(db)
})
AfterEach(func() {
test_helpers.TearDown(db)
})
var fakeAddress = "0x1234567890abcdef"
Describe("SetEvents", func() {
It("Sets which events to watch from the given contract address", func() {
watchedEvents := []string{"Transfer", "Mint"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, watchedEvents)
Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, watchedEvents)
Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
})
})
Describe("SetEventAddrs", func() {
It("Sets which account addresses to watch events for", func() {
eventAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEventArgs(fakeAddress, eventAddrs)
Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
})
})
Describe("SetMethods", func() {
It("Sets which methods to poll at the given contract address", func() {
watchedMethods := []string{"balanceOf", "totalSupply"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethods(constants.TusdContractAddress, watchedMethods)
Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethods(fakeAddress, watchedMethods)
Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
})
})
Describe("SetMethodAddrs", func() {
It("Sets which account addresses to poll methods against", func() {
methodAddrs := []string{"test1", "test2"}
t := transformer.NewTransformer("", blockChain, db)
t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetMethodArgs(fakeAddress, methodAddrs)
Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
})
})
Describe("SetStartingBlock", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetStartingBlock(constants.TusdContractAddress, 11)
Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetStartingBlock(fakeAddress, 11)
Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
})
})
Describe("SetCreateAddrList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateAddrList(constants.TusdContractAddress, true)
Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateAddrList(fakeAddress, true)
Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
})
})
Describe("SetCreateHashList", func() {
It("Sets the block range that the contract should be watched within", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetCreateHashList(constants.TusdContractAddress, true)
Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetCreateHashList(fakeAddress, true)
Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
})
})
Describe("Init", func() {
It("Initializes transformer's contract objects", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockLightBlockRetriever{}
firstBlock := int64(1)
blockRetriever.FirstBlock = firstBlock
parsr := &fakes.MockParser{}
fakeAbi := "fake_abi"
parsr.AbiToReturn = fakeAbi
pollr := &fakes.MockPoller{}
fakeContractName := "fake_contract_name"
pollr.ContractName = fakeContractName
t := getFakeTransformer(blockRetriever, parsr, pollr)
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
c, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(true))
Expect(c.StartingBlock).To(Equal(int64(6194632)))
Expect(c.StartingBlock).To(Equal(firstBlock))
Expect(c.LastBlock).To(Equal(int64(-1)))
Expect(c.Abi).To(Equal(constants.TusdAbiString))
Expect(c.Name).To(Equal("TrueUSD"))
Expect(c.Address).To(Equal(tusdAddr))
Expect(c.Abi).To(Equal(fakeAbi))
Expect(c.Name).To(Equal(fakeContractName))
Expect(c.Address).To(Equal(fakeAddress))
})
It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
err = t.Init()
blockRetriever := &fakes.MockLightBlockRetriever{}
blockRetriever.FirstBlockErr = fakes.FakeError
t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
t.SetEvents(fakeAddress, []string{"Transfer"})
err := t.Init()
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("Does nothing if watched events are unset", func() {
headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
t := transformer.NewTransformer("", blockChain, db)
err = t.Init()
t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
err := t.Init()
Expect(err).ToNot(HaveOccurred())
_, ok := t.Contracts[tusdAddr]
_, ok := t.Contracts[fakeAddress]
Expect(ok).To(Equal(false))
})
})
Describe("Execute- against TrueUSD contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(log.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedAddrs)).To(Equal(4))
Expect(len(c.EmittedHashes)).To(Equal(0))
b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[""]
Expect(ok).To(Equal(false))
_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
Expect(ok).To(Equal(false))
})
It("Polls given methods using generated token holder address", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Balance).To(Equal("55849938025000000000000"))
Expect(res.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("Fails if initialization has not been done", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Execute()
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against ENS registry contract", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(log.HeaderID).To(Equal(headerID))
Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
})
It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
c, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(c.EmittedHashes)).To(Equal(2))
Expect(len(c.EmittedAddrs)).To(Equal(0))
b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
Expect(ok).To(Equal(false))
})
It("Polls given method using list of collected hashes", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
It("It does not persist events if they do not pass the emitted arg filter", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
log := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
Expect(err).To(HaveOccurred())
})
It("If a method arg filter is applied, only those arguments are used in polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
res := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).ToNot(HaveOccurred())
Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(res.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
Expect(err).To(HaveOccurred())
})
})
Describe("Execute- against both ENS and TrueUSD", func() {
BeforeEach(func() {
header1, err := blockChain.GetHeaderByNumber(6791668)
Expect(err).ToNot(HaveOccurred())
header2, err := blockChain.GetHeaderByNumber(6791669)
Expect(err).ToNot(HaveOccurred())
header3, err := blockChain.GetHeaderByNumber(6791670)
Expect(err).ToNot(HaveOccurred())
header4, err := blockChain.GetHeaderByNumber(6885695)
Expect(err).ToNot(HaveOccurred())
header5, err := blockChain.GetHeaderByNumber(6885696)
Expect(err).ToNot(HaveOccurred())
header6, err := blockChain.GetHeaderByNumber(6885697)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header1)
headerID, err = headerRepository.CreateOrUpdateHeader(header2)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header3)
headerRepository.CreateOrUpdateHeader(header4)
headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
Expect(err).ToNot(HaveOccurred())
headerRepository.CreateOrUpdateHeader(header6)
})
It("Transforms watched contract data into custom repositories", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, nil)
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, nil)
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
newOwnerLog := test_helpers.LightNewOwnerLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
transferLog := test_helpers.LightTransferLog{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
Expect(err).ToNot(HaveOccurred())
// We don't know vulcID, so compare individual fields instead of complete structures
Expect(transferLog.HeaderID).To(Equal(headerID))
Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
Expect(transferLog.Value).To(Equal("9998940000000000000000"))
})
It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
ens, ok := t.Contracts[ensAddr]
Expect(ok).To(Equal(true))
tusd, ok := t.Contracts[tusdAddr]
Expect(ok).To(Equal(true))
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
Expect(len(ens.EmittedHashes)).To(Equal(2))
Expect(len(ens.EmittedAddrs)).To(Equal(0))
Expect(len(tusd.EmittedAddrs)).To(Equal(4))
Expect(len(tusd.EmittedHashes)).To(Equal(0))
b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
Expect(ok).To(Equal(true))
Expect(b).To(Equal(true))
})
It("Polls given methods for each contract, using list of collected values", func() {
t := transformer.NewTransformer("", blockChain, db)
t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
t.SetMethods(constants.EnsContractAddress, []string{"owner"})
t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
err = t.Init()
Expect(err).ToNot(HaveOccurred())
err = t.Execute()
Expect(err).ToNot(HaveOccurred())
owner := test_helpers.Owner{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).ToNot(HaveOccurred())
Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
Expect(owner.TokenName).To(Equal(""))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
Expect(err).To(HaveOccurred())
bal := test_helpers.BalanceOf{}
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).ToNot(HaveOccurred())
Expect(bal.Balance).To(Equal("55849938025000000000000"))
Expect(bal.TokenName).To(Equal("TrueUSD"))
err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
Expect(err).To(HaveOccurred())
})
})
})
func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
return transformer.Transformer{
Parser: parsr,
BlockRetriever: blockRetriever,
Poller: pollr,
HeaderRepository: &fakes.MockLightHeaderRepository{},
Contracts: map[string]*contract.Contract{},
WatchedEvents: map[string][]string{},
WantedMethods: map[string][]string{},
ContractStart: map[string]int64{},
EventArgs: map[string][]string{},
MethodArgs: map[string][]string{},
CreateAddrList: map[string]bool{},
CreateHashList: map[string]bool{},
}
}

1
pkg/omni/shared/constants/constants.go
View File

@ -72,6 +72,7 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b"
var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5"
// TODO: Consider whether these should be moved to plugins
// Contract Owner
var DaiContractOwner = "0x0000000000000000000000000000000000000000"
var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f"

12
pkg/omni/shared/helpers/test_helpers/database.go
View File

@ -106,6 +106,7 @@ type Owner struct {
Address string `db:"returned"`
}
// TODO: consider whether this should be moved to libraries/shared
func SetupBC() core.BlockChain {
infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
rawRpcClient, err := rpc.Dial(infuraIPC)
@ -113,9 +114,9 @@ func SetupBC() core.BlockChain {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
blockChainNode := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
return blockChain
}
@ -127,9 +128,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
blockChainNode := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
@ -188,6 +189,7 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract
}.Init()
}
// TODO: consider whether this can be moved to plugin or libraries/shared
func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) {
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",
@ -236,7 +238,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
}
func TearDown(db *postgres.DB) {
tx, err := db.Begin()
tx, err := db.Beginx()
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM blocks`)

4
pkg/omni/shared/repository/event_repository.go
View File

@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
// Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Begin()
tx, err := r.db.Beginx()
if err != nil {
return err
}
@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
// Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
tx, err := r.db.Begin()
tx, err := r.db.Beginx()
if err != nil {
return err
}

2
pkg/omni/shared/repository/method_repository.go
View File

@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ
// Creates a custom postgres command to persist logs for the given event
func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
tx, err := r.DB.Begin()
tx, err := r.DB.Beginx()
if err != nil {
return err
}

5
pkg/plugin/test_helpers/database.go
View File

@ -31,6 +31,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/geth/node"
)
// TODO: consider whether this should be moved to libraries/shared
func SetupDBandBC() (*postgres.DB, core.BlockChain) {
infuraIPC := "http://kovan0.vulcanize.io:8545"
rawRpcClient, err := rpc.Dial(infuraIPC)
@ -38,9 +39,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
ethClient := ethclient.NewClient(rawRpcClient)
blockChainClient := client.NewEthClient(ethClient)
node := node.MakeNode(rpcClient)
blockChainNode := node.MakeNode(rpcClient)
transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
db, err := postgres.NewDB(config.Database{
Hostname: "localhost",

2
scripts/start_blockchain
View File

@ -1,2 +0,0 @@
#!/bin/bash
geth --fast --cache=1024

10
scripts/start_test_chain.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
MNEMONIC_PHRASE="whisper ordinary mystery awesome wood fox auction february blind volcano spare soft"
PORT=7545
DATABASE_PATH=test_data/test_chain/
echo Starting ganache chain on port $PORT...
ganache-cli --port $PORT \
--db $DATABASE_PATH \
2>&1 > ganache-output.log &

4
scripts/stop_test_chain.sh
View File

@ -1,4 +0,0 @@
#!/bin/bash
echo "Stopping ganache chain on port 7545"
ps -ef | grep ganache | grep -v grep | awk '{print $2}' | xargs kill

5
test_config/test_config.go
View File

@ -95,10 +95,13 @@ func NewTestDB(node core.Node) *postgres.DB {
func CleanTestDB(db *postgres.DB) {
db.MustExec("DELETE FROM blocks")
db.MustExec("DELETE FROM headers")
db.MustExec("DELETE FROM checked_headers")
// can't delete from eth_nodes since this function is called after the required eth_node is persisted
db.MustExec("DELETE FROM goose_db_version")
db.MustExec("DELETE FROM headers")
db.MustExec("DELETE FROM log_filters")
db.MustExec("DELETE FROM logs")
db.MustExec("DELETE FROM queued_storage")
db.MustExec("DELETE FROM receipts")
db.MustExec("DELETE FROM transactions")
db.MustExec("DELETE FROM watched_contracts")