Merge pull request #26 from vulcanize/vdb-398-extract-slow-tests

Vdb 398 extract slow tests

.private_blockchain_password

@@ -1 +0,0 @@
-1234

.travis.yml

@@ -6,26 +6,20 @@ services:
   - postgresql
 addons:
   postgresql: "9.6"
-  ssh_known_hosts:
-    - 147.75.96.51
 
 go_import_path: github.com/vulcanize/vulcanizedb
 
 before_install:
   # ginkgo golint dep goose
-  - echo -e "Host github.com\n\tHostName github.com\n\tUser git\n\tIdentityFile ~/.ssh/id_rsa\n" >> ~/.ssh/config
   - make installtools
   - bash ./scripts/install-postgres-10.sh
-  - npm install -g ganache-cli
   - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
   - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
   - sudo apt-get update && sudo apt-get install yarn
 
 before_script:
-  - go get -u github.com/pressly/sup/cmd/sup
   - sudo -u postgres createdb vulcanize_private
   - make migrate NAME=vulcanize_private
-  - bash ./scripts/start_test_chain.sh
   - cd postgraphile && yarn
 
 script:
@@ -36,7 +30,3 @@ script:
 
 notifications:
   email: false
-
-after_script:
-  - bash ./scripts/stop_test_chain.sh
-  - bash ./bin/deploy.sh

README.md

@@ -79,7 +79,7 @@ Lastly, ensure that `GOPATH` is defined in your shell. If necessary, `GOPATH` ca
         - The default location is:
           - Mac: `<full home path>/Library/Application\ Support/io.parity.ethereum/`
           - Linux: `<full home path>/local/share/io.parity.ethereum/`
-          
+
   - `levelDbPath` should match Geth's chaindata directory path.
       - The geth LevelDB chaindata path is printed to the console when you start geth.
       - The default location is:
@@ -112,8 +112,8 @@ Syncs VulcanizeDB with the configured Ethereum node, populating only block heade
 This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values).
 1. Start Ethereum node
 1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`  
+    - `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`
-  
+
 ## Start full environment in docker by single command
 
 ### Geth Rinkeby
@@ -150,68 +150,68 @@ If you have full rinkeby chaindata you can move it to `rinkeby_vulcanizedb_geth_
 
 ## Contract Watchers
 Contract watchers work with a light or full sync vDB to fetch raw ethereum data and execute a set of transformations over them, persisting the output.    
-  
+
 A watcher is composed of at least a fetcher and a transformer or set of transformers, where a fetcher is an interface for retrieving raw Ethereum data from some source (e.g. eth_jsonrpc, IPFS)
 and a transformer is an interface for filtering through that raw Ethereum data to extract, process, and persist data for specific contracts or accounts. 
 
-### omniWatcher 
+### omniWatcher
 The `omniWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available. 
-It also provides some state variable coverage by automating polling of public methods, with some restrictions.   
+It also provides some state variable coverage by automating polling of public methods, with some restrictions.
 
 This command requires a pre-synced (full or light) vulcanizeDB (see above sections) and currently requires the contract ABI be available on etherscan or provided by the user.   
- 
+
-To watch all events of a contract using a light synced vDB:  
+To watch all events of a contract using a light synced vDB:
-    - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address>`  
+    - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address>`
-    
+
-Or if you are using a full synced vDB, change the mode to full:  
+Or if you are using a full synced vDB, change the mode to full:
-    - Execute `./vulcanizedb omniWatcher --mode full --config <path to config.toml> --contract-address <contract address>`  
+    - Execute `./vulcanizedb omniWatcher --mode full --config <path to config.toml> --contract-address <contract address>`
-    
+
-To watch contracts on a network other than mainnet, use the network flag:  
+To watch contracts on a network other than mainnet, use the network flag:
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --network <ropsten, kovan, or rinkeby>`  
-    
+
 To watch events starting at a certain block use the starting block flag:
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --starting-block-number <#>`
-    
+
-To watch only specified events use the events flag:  
+To watch only specified events use the events flag:
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --events <EventName1> --events <EventName2>`  
-    
+
 To watch events and poll the specified methods with any addresses and hashes emitted by the watched events utilize the methods flag:  
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --methods <methodName1> --methods <methodName2>`  
-    
+
-To watch specified events and poll the specified method with any addresses and hashes emitted by the watched events:  
+To watch specified events and poll the specified method with any addresses and hashes emitted by the watched events:
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --events <EventName1> --events <EventName2> --methods <methodName>`  
-    
+
 To turn on method piping so that values returned from previous method calls are cached and used as arguments in subsequent method calls:  
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --piping true --contract-address <contract address> --events <EventName1> --events <EventName2> --methods <methodName>`  
-    
+
-To watch all types of events of the contract but only persist the ones that emit one of the filtered-for argument values:  
+To watch all types of events of the contract but only persist the ones that emit one of the filtered-for argument values:
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --event-args <arg1> --event-args <arg2>`  
-    
+
 To watch all events of the contract but only poll the specified method with specified argument values (if they are emitted from the watched events):  
     - Execute `./vulcanizedb omniWatcher --config <path to config.toml> --contract-address <contract address> --methods <methodName> --method-args <arg1> --method-args <arg2>`  
 
-#### omniWatcher output    
+#### omniWatcher output
 
 Transformed events and polled method results are committed to Postgres in schemas and tables generated according to the contract abi.      
 
-Schemas are created for each contract using the naming convention `<sync-type>_<lowercase contract-address>`    
+Schemas are created for each contract using the naming convention `<sync-type>_<lowercase contract-address>`
 Under this schema, tables are generated for watched events as `<lowercase event name>_event` and for polled methods as `<lowercase method name>_method`  
 The 'method' and 'event' identifiers are tacked onto the end of the table names to prevent collisions between methods and events of the same lowercase name    
 
-Example: 
+Example:
 
 Running `./vulcanizedb omniWatcher --config <path to config> --starting-block-number=5197514 --contract-address=0x8dd5fbce2f6a956c3022ba3663759011dd51e73e --events=Transfer --events=Mint --methods=balanceOf` 
 watches Transfer and Mint events of the TrueUSD contract and polls its balanceOf method using the addresses we find emitted from those events    
 
-It produces and populates a schema with three tables:   
+It produces and populates a schema with three tables:
 
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`  
+`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`  
+`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`  
+`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
 
 Column ids and types for these tables are generated based on the event and method argument names and types and method return types, resulting in tables such as:
 
-Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"  
+Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
 
 |  Column    |         Type          | Collation | Nullable |                                           Default                                           | Storage  | Stats target | Description  
 |:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
@@ -224,9 +224,9 @@ Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
 | from_      | character varying(66) |           | not null |                                                                                             | extended |              |             |
 | to_        | character varying(66) |           | not null |                                                                                             | extended |              |             |
 | value_     | numeric               |           | not null |                                                                                             | main     |              |             |
-     
 
-Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"  
+
+Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
 
 |   Column   |         Type          | Collation | Nullable |                                            Default                                            | Storage  | Stats target | Description |
 |:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
@@ -235,32 +235,32 @@ Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
 | block      | integer               |           | not null |                                                                                               | plain    |              |             |
 | who_       | character varying(66) |           | not null |                                                                                               | extended |              |             |
 | returned   | numeric               |           | not null |                                                                                               | main     |              |             |
-    
+  
-The addition of '_' after table names is to prevent collisions with reserved Postgres words   
+The addition of '_' after table names is to prevent collisions with reserved Postgres words
 
-### composeAndExecute 
+### composeAndExecute
 The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers.
 This is accomplished by generating a Go pluggin which allows our `vulcanizedb` binary to link to external transformers, so
 long as they abide by our standard [interfaces](https://github.com/vulcanize/maker-vulcanizedb/tree/compose_and_execute/libraries/shared/transformer).   
 
-This command requires Go 1.11+ and [Go plugins](https://golang.org/pkg/plugin/) only work on Unix based systems.   
+This command requires Go 1.11+ and [Go plugins](https://golang.org/pkg/plugin/) only work on Unix based systems.
 
-#### Writing custom transformers 
+#### Writing custom transformers
-Storage Transformers   
+Storage Transformers
    * [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/README.md)   
    * [Example](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/EXAMPLE.md)   
-    
+
-Event Transformers   
+Event Transformers
    * Guide
    * Example
-   
+
 #### composeAndExecute configuration
-A .toml config file is specified when executing the command:  
+A .toml config file is specified when executing the command:
-`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`  
+`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
 
-The config provides information for composing a set of transformers:         
+The config provides information for composing a set of transformers:
 
-```toml 
+```toml
 [database]
     name     = "vulcanize_public"
     hostname = "localhost"
@@ -316,14 +316,14 @@ The config provides information for composing a set of transformers:
         - `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go)
          that fetches event logs from an ETH node
     - `migrations` is the relative path from `repository` to the db migrations directory for the transformer
-- Note: If any of the imported transformers need additional config variables those need to be included as well   
+- Note: If any of the imported transformers need additional config variables those need to be included as well
 
 This information is used to write and build a Go plugin which exports the configured transformers.
-These transformers are loaded onto their specified watchers and executed.   
+These transformers are loaded onto their specified watchers and executed.
 
 Transformers of different types can be run together in the same command using a single config file or in separate instances using different config files   
 
-The general structure of a plugin .go file, and what we would see built with the above config is shown below   
+The general structure of a plugin .go file, and what we would see built with the above config is shown below
 
 ```go
 package main
@@ -352,7 +352,7 @@ func (e exporter) Export() []interface1.TransformerInitializer, []interface1.Sto
 ```
 
 #### Preparing transformer(s) to work as pluggins for composeAndExecute
-To plug in an external transformer we need to:   
+To plug in an external transformer we need to:
 
 * Create a [package](https://github.com/vulcanize/mcd_transformers/blob/staging/transformers/bite/initializer/initializer.go) 
 that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33)
@@ -361,7 +361,7 @@ or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanized
 or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53) watchers
 * Create db migrations to run against vulcanizeDB so that we can store the transformer output
     * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
-    * Do not `goose fix` the transformer migrations   
+    * Do not `goose fix` the transformer migrations
 
 To update a plugin repository with changes to the core vulcanizedb repository, replace the vulcanizedb vendored in the plugin repo (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb`)
 with the newly updated version

Supfile

@@ -1,64 +0,0 @@
----
-version: 0.5
-
-env:
-  VDB_PATH: /root/go_projects/src/github.com/vulcanize/vulcanizedb
-  VDB_PG_USER: vulcanize
-  VDB_PG_PW: vulcanize
-
-networks:
-  staging:
-    hosts:
-      - root@147.75.96.51
-  prod:
-    hosts:
-      - root@147.75.197.13
-
-targets:
-  deploy:
-    - remove
-    - transfer
-    - buildPostgraphile
-    - buildVDB
-    - migrate
-    - lightSync
-    - postgraphile
-
-commands:
-  remove:
-    desc: remove old vulcanizedb
-    run: rm -rf $VDB_PATH && rm -rf /usr/local/vulcanizedb && mkdir -p $VDB_PATH
-  transfer:
-    desc: transfer repo to remote server
-    upload:
-      - src: .
-        dst: $VDB_PATH
-  migrate:
-    desc: run migration
-    run: >
-      cd $VDB_PATH &&
-      make installtools &&
-      cd db/migrations &&
-      /root/go_projects/bin/goose postgres "postgresql://$(VDB_PG_USER):$(VDB_PG_PW)@127.0.0.1:5432/vulcanize_public?sslmode=disable" up
-  buildPostgraphile:
-    desc: build postgraphile app
-    run: >
-      cd $VDB_PATH/postgraphile &&
-      yarn && tsc
-  buildVDB:
-    desc: build vulcanizedb
-    run: >
-      cd $VDB_PATH &&
-      GOPATH=$HOME/go_projects go get &&
-      GOPATH=$HOME/go_projects go build &&
-      cp -r . /usr/local/vulcanizedb
-  lightSync:
-    desc: start vdb light sync
-    run: >
-      systemctl daemon-reload &&
-      sudo systemctl restart vulcanizedb_light_sync.service &&
-      sudo systemctl restart vulcanizedb_log_sync.service &&
-      sudo systemctl restart vulcanizedb_recheck_sync.service
-  postgraphile:
-    desc: start postgraphile
-    run: systemctl daemon-reload && sudo systemctl restart postgraphile.service

bin/deploy.sh

@@ -1,5 +0,0 @@
-if [ $TRAVIS_BRANCH == 'staging' ]; then
-  sup --debug staging deploy
-elif [ $TRAVIS_BRANCH == 'master' ]; then
-  sup --debug prod deploy
-fi

integration_test/omni_full_transformer_test.go

@@ -0,0 +1,275 @@
+package integration
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+)
+
+var _ = Describe("Omni full transformer", func() {
+	var db *postgres.DB
+	var err error
+	var blockChain core.BlockChain
+	var blockRepository repositories.BlockRepository
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+	rand.Seed(time.Now().UnixNano())
+
+	BeforeEach(func() {
+		db, blockChain = test_helpers.SetupDBandBC()
+		blockRepository = *repositories.NewBlockRepository(db)
+	})
+
+	AfterEach(func() {
+		test_helpers.TearDown(db)
+	})
+
+	Describe("Init", func() {
+		It("Initializes transformer's contract objects", func() {
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+
+			Expect(c.StartingBlock).To(Equal(int64(6194633)))
+			Expect(c.LastBlock).To(Equal(int64(6194634)))
+			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Address).To(Equal(tusdAddr))
+		})
+	})
+
+	Describe("Execute", func() {
+		BeforeEach(func() {
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.TransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
+
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
+			Expect(log.Block).To(Equal(int64(6194634)))
+			Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
+			Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
+			Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
+		})
+
+		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[""]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.BalanceOf{}
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Balance).To(Equal("0"))
+			Expect(res.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Balance).To(Equal("0"))
+			Expect(res.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Fails if initialization has not been done", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+
+			err = t.Execute()
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Execute- against ENS registry contract", func() {
+		BeforeEach(func() {
+			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
+			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.NewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
+
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
+			Expect(log.Block).To(Equal(int64(6194635)))
+			Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
+			Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
+			Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
+		})
+
+		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[ensAddr]
+			Expect(ok).To(Equal(true))
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedHashes)).To(Equal(3))
+
+			b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("It does not perist events if they do not pass the emitted arg filter", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
+
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("If a method arg filter is applied, only those arguments are used in polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

integration_test/omni_light_transformer_test.go

@@ -0,0 +1,413 @@
+package integration
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+)
+
+var _ = Describe("Omnit light transformer", func() {
+	var db *postgres.DB
+	var err error
+	var blockChain core.BlockChain
+	var headerRepository repositories.HeaderRepository
+	var headerID, headerID2 int64
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+
+	BeforeEach(func() {
+		db, blockChain = test_helpers.SetupDBandBC()
+		headerRepository = repositories.NewHeaderRepository(db)
+	})
+
+	AfterEach(func() {
+		test_helpers.TearDown(db)
+	})
+
+	Describe("Init", func() {
+		It("Initializes transformer's contract objects", func() {
+			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+
+			Expect(c.StartingBlock).To(Equal(int64(6194632)))
+			Expect(c.LastBlock).To(Equal(int64(-1)))
+			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Address).To(Equal(tusdAddr))
+		})
+	})
+
+	Describe("Execute- against TrueUSD contract", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6791668)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6791669)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6791670)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.HeaderID).To(Equal(headerID))
+			Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
+			Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
+			Expect(log.Value).To(Equal("9998940000000000000000"))
+		})
+
+		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedAddrs)).To(Equal(4))
+			Expect(len(c.EmittedHashes)).To(Equal(0))
+
+			b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[""]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.BalanceOf{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Balance).To(Equal("55849938025000000000000"))
+			Expect(res.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Fails if initialization has not been done", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Execute()
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Execute- against ENS registry contract", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6885695)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6885696)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6885697)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.HeaderID).To(Equal(headerID))
+			Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
+			Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
+			Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+		})
+
+		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			c, ok := t.Contracts[ensAddr]
+			Expect(ok).To(Equal(true))
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedHashes)).To(Equal(2))
+			Expect(len(c.EmittedAddrs)).To(Equal(0))
+
+			b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given method using list of collected hashes", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("It does not persist events if they do not pass the emitted arg filter", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("If a method arg filter is applied, only those arguments are used in polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Execute- against both ENS and TrueUSD", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6791668)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6791669)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6791670)
+			Expect(err).ToNot(HaveOccurred())
+			header4, err := blockChain.GetHeaderByNumber(6885695)
+			Expect(err).ToNot(HaveOccurred())
+			header5, err := blockChain.GetHeaderByNumber(6885696)
+			Expect(err).ToNot(HaveOccurred())
+			header6, err := blockChain.GetHeaderByNumber(6885697)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+			headerRepository.CreateOrUpdateHeader(header4)
+			headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header6)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			newOwnerLog := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
+			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
+			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
+			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+
+			transferLog := test_helpers.LightTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(transferLog.HeaderID).To(Equal(headerID))
+			Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
+			Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
+			Expect(transferLog.Value).To(Equal("9998940000000000000000"))
+		})
+
+		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			ens, ok := t.Contracts[ensAddr]
+			Expect(ok).To(Equal(true))
+			tusd, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(ens.EmittedHashes)).To(Equal(2))
+			Expect(len(ens.EmittedAddrs)).To(Equal(0))
+			Expect(len(tusd.EmittedAddrs)).To(Equal(4))
+			Expect(len(tusd.EmittedHashes)).To(Equal(0))
+
+			b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+		})
+
+		It("Polls given methods for each contract, using list of collected values", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			owner := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(owner.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(owner.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			Expect(err).To(HaveOccurred())
+
+			bal := test_helpers.BalanceOf{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal.Balance).To(Equal("55849938025000000000000"))
+			Expect(bal.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

libraries/shared/README.md

@@ -0,0 +1,8 @@
+# Shared Tools
+
+## Description
+Code that is useful for or used by plugins written on top of VulcanizeDB.
+
+## Note
+Much code in this directory may not be used outside of the tests, but don't delete it - it could be used by a plugin.
+Renaming and/or deleting functions in this namespace requires a version bump to avoid breaking plugins.

libraries/shared/chunker/log_chunker_test.go

@@ -58,9 +58,9 @@ var _ = Describe("Log chunker", func() {
 	Describe("initialisation", func() {
 		It("creates lookup maps correctly", func() {
 			Expect(chunker.AddressToNames).To(Equal(map[string][]string{
-				"0x00000000000000000000000000000000000000a1": []string{"TransformerA"},
+				"0x00000000000000000000000000000000000000a1": {"TransformerA"},
-				"0x00000000000000000000000000000000000000a2": []string{"TransformerA", "TransformerC"},
+				"0x00000000000000000000000000000000000000a2": {"TransformerA", "TransformerC"},
-				"0x00000000000000000000000000000000000000b1": []string{"TransformerB"},
+				"0x00000000000000000000000000000000000000b1": {"TransformerB"},
 			}))
 
 			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{

libraries/shared/constants/data.go

@@ -17,6 +17,3 @@
 package constants
 
 var DataItemLength = 32
-
-// TODO Grab this from DB, since it can change through governance
-var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours

libraries/shared/repository/repository.go

@@ -18,9 +18,9 @@ package repository
 
 import (
 	"bytes"
-	"database/sql"
 	"database/sql/driver"
 	"fmt"
+	"github.com/jmoiron/sqlx"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -35,7 +35,7 @@ func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn str
 	return err
 }
 
-func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
+func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
 	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
 		VALUES ($1, $2)
 		ON CONFLICT (header_id) DO
@@ -155,14 +155,3 @@ func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.Transfor
 
 	return result.String()
 }
-
-func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) {
-	var blockTimestamp int64
-	err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp)
-	if err != nil {
-		return 0, err
-	}
-
-	tic := blockTimestamp + constants.TTL
-	return tic, nil
-}

libraries/shared/repository/repository_test.go

@@ -19,6 +19,7 @@ package repository_test
 import (
 	"fmt"
 	"math/rand"
+	"strconv"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -34,7 +35,83 @@ import (
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("Repository utilities", func() {
+var _ = Describe("Repository", func() {
+	Describe("MarkHeaderChecked", func() {
+		var (
+			checkedHeadersColumn string
+			db                   *postgres.DB
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+
+			checkedHeadersColumn = "test_column_checked"
+			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
+				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
+			Expect(migrateErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+		})
+
+		It("marks passed column as checked for passed header", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(headerErr).NotTo(HaveOccurred())
+
+			err := shared.MarkHeaderChecked(headerID, db, checkedHeadersColumn)
+
+			Expect(err).NotTo(HaveOccurred())
+			var checkedCount int
+			fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
+			Expect(fetchErr).NotTo(HaveOccurred())
+			Expect(checkedCount).To(Equal(1))
+		})
+	})
+
+	Describe("MarkHeaderCheckedInTransaction", func() {
+		var (
+			checkedHeadersColumn string
+			db                   *postgres.DB
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+
+			checkedHeadersColumn = "test_column_checked"
+			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
+				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
+			Expect(migrateErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+		})
+
+		It("marks passed column as checked for passed header within a passed transaction", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(headerErr).NotTo(HaveOccurred())
+			tx, txErr := db.Beginx()
+			Expect(txErr).NotTo(HaveOccurred())
+
+			err := shared.MarkHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
+
+			Expect(err).NotTo(HaveOccurred())
+			commitErr := tx.Commit()
+			Expect(commitErr).NotTo(HaveOccurred())
+			var checkedCount int
+			fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
+			Expect(fetchErr).NotTo(HaveOccurred())
+			Expect(checkedCount).To(Equal(1))
+		})
+	})
+
 	Describe("MissingHeaders", func() {
 		var (
 			db                       *postgres.DB
@@ -116,6 +193,84 @@ var _ = Describe("Repository utilities", func() {
 		})
 	})
 
+	Describe("RecheckHeaders", func() {
+		var (
+			checkedHeadersColumn                                      string
+			db                                                        *postgres.DB
+			headerOneID, headerTwoID, headerThreeID, headerFourID     int64
+			headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+
+			// create header checked column
+			checkedHeadersColumn = "test_column_checked"
+			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
+			Expect(migrateErr).NotTo(HaveOccurred())
+
+			// create headers
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
+			Expect(headerOneErr).NotTo(HaveOccurred())
+			headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
+			Expect(headerTwoErr).NotTo(HaveOccurred())
+			headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
+			Expect(headerThreeErr).NotTo(HaveOccurred())
+			headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
+			Expect(headerFourErr).NotTo(HaveOccurred())
+
+			// mark every header checked at least once, with one fully rechecked (headerThree)
+			maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
+			Expect(intConversionErr).NotTo(HaveOccurred())
+			_, markHeaderOneCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerOneID, maxCheckCount)
+			Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
+			_, markHeaderTwoCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerTwoID, maxCheckCount)
+			Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
+			_, markHeaderThreeCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerThreeID, maxCheckCount+1)
+			Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
+			_, markHeaderFourCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerFourID, maxCheckCount)
+			Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+		})
+
+		Describe("when no ending block number (ending block number == -1)", func() {
+			It("returns all headers since starting block where checked count is less than cap", func() {
+				headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(3))
+				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+				Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+			})
+		})
+
+		Describe("when ending block number specified", func() {
+			It("returns headers between starting and ending block where checked count is less than cap", func() {
+				headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(2))
+				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
+				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
+			})
+		})
+	})
+
 	Describe("GetCheckedColumnNames", func() {
 		It("gets the column names from checked_headers", func() {
 			db := test_config.NewTestDB(test_config.NewTestNode())
@@ -162,33 +317,9 @@ var _ = Describe("Repository utilities", func() {
 
 func getExpectedColumnNames() []string {
 	return []string{
-		"price_feeds_checked",
+		"column_1_checked",
-		"flip_kick_checked",
+		"column_2_checked",
-		"frob_checked",
+		"column_3_checked",
-		"tend_checked",
+		"column_4_checked",
-		"bite_checked",
-		"dent_checked",
-		"pit_file_debt_ceiling_checked",
-		"pit_file_ilk_checked",
-		"vat_init_checked",
-		"drip_file_ilk_checked",
-		"drip_file_repo_checked",
-		"drip_file_vow_checked",
-		"deal_checked",
-		"drip_drip_checked",
-		"cat_file_chop_lump_checked",
-		"cat_file_flip_checked",
-		"cat_file_pit_vow_checked",
-		"flop_kick_checked",
-		"vat_move_checked",
-		"vat_fold_checked",
-		"vat_heal_checked",
-		"vat_toll_checked",
-		"vat_tune_checked",
-		"vat_grab_checked",
-		"vat_flux_checked",
-		"vat_slip_checked",
-		"vow_flog_checked",
-		"flap_kick_checked",
 	}
 }

libraries/shared/storage/mappings_test.go

@@ -0,0 +1,65 @@
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+)
+
+var _ = Describe("Mappings", func() {
+	Describe("GetMapping", func() {
+		It("returns the storage key for a mapping when passed the mapping's index on the contract and the desired value's key", func() {
+			// ex. solidity:
+			//    	mapping (bytes32 => uint) public amounts
+			// to access amounts, pass in the index of the mapping on the contract + the bytes32 key for the uint val being looked up
+			indexOfMappingOnContract := storage.IndexZero
+			keyForDesiredValueInMapping := "1234567890abcdef"
+
+			storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+
+			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
+	})
+
+	Describe("GetNestedMapping", func() {
+		It("returns the storage key for a nested mapping when passed the mapping's index on the contract and the desired value's keys", func() {
+			// ex. solidity:
+			//    	mapping (bytes32 => uint) public amounts
+			//    	mapping (address => mapping (uint => bytes32)) public addressNames
+			// to access addressNames, pass in the index of the mapping on the contract + the address and uint keys for the bytes32 val being looked up
+			indexOfMappingOnContract := storage.IndexOne
+			keyForOuterMapping := "1234567890abcdef"
+			keyForInnerMapping := "123"
+
+			storageKey := storage.GetNestedMapping(indexOfMappingOnContract, keyForOuterMapping, keyForInnerMapping)
+
+			expectedStorageKey := common.HexToHash("0x82113529f6cd61061d1a6f0de53f2bdd067a1addd3d2b46be50a99abfcdb1661")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
+	})
+
+	Describe("GetIncrementedKey", func() {
+		It("returns the storage key for later values sharing an index on the contract with other earlier values", func() {
+			// ex. solidity:
+			//    	mapping (bytes32 => uint) public amounts
+			//    	mapping (address => mapping (uint => bytes32)) public addressNames
+			//    	struct Data {
+			//        uint256 quantity;
+			//        uint256 quality;
+			//    	}
+			//    	mapping (bytes32 => Data) public itemData;
+			// to access quality from itemData, pass in the storage key for the zero-indexed value (quantity) + the number of increments required.
+			// (For "quality", we must increment the storage key for the corresponding "quantity" by 1).
+			indexOfMappingOnContract := storage.IndexTwo
+			keyForDesiredValueInMapping := "1234567890abcdef"
+			storageKeyForFirstPropertyOnStruct := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+
+			storageKey := storage.GetIncrementedKey(storageKeyForFirstPropertyOnStruct, 1)
+
+			expectedStorageKey := common.HexToHash("0x69b38749f0a8ed5d505c8474f7fb62c7828aad8a7627f1c67e07af1d2368cad4")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
+	})
+})

libraries/shared/storage/utils/value_test.go

@@ -0,0 +1,22 @@
+package utils_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
+
+var _ = Describe("Storage value metadata getter", func() {
+	It("returns a storage value metadata instance with corresponding fields assigned", func() {
+		metadataName := "fake_name"
+		metadataKeys := map[utils.Key]string{"key": "value"}
+		metadataType := utils.Uint256
+
+		expectedMetadata := utils.StorageValueMetadata{
+			Name: metadataName,
+			Keys: metadataKeys,
+			Type: metadataType,
+		}
+		Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
+	})
+})

libraries/shared/test_data/generic.go

@@ -49,7 +49,7 @@ var GenericTestConfig = transformer.TransformerConfig{
 }
 
 func randomString(length int) string {
-	var seededRand *rand.Rand = rand.New(
+	var seededRand = rand.New(
 		rand.NewSource(time.Now().UnixNano()))
 	charset := "abcdefghijklmnopqrstuvwxyz1234567890"
 	b := make([]byte, length)

libraries/shared/watcher/storage_watcher.go

@@ -50,8 +50,8 @@ func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
 
 func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
-		transformer := initializer(watcher.db)
+		storageTransformer := initializer(watcher.db)
-		watcher.Transformers[transformer.ContractAddress()] = transformer
+		watcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
 	}
 }
 
@@ -65,12 +65,12 @@ func (watcher StorageWatcher) Execute() error {
 		if parseErr != nil {
 			return parseErr
 		}
-		transformer, ok := watcher.Transformers[row.Contract]
+		storageTransformer, ok := watcher.Transformers[row.Contract]
 		if !ok {
 			logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
 			continue
 		}
-		executeErr := transformer.Execute(row)
+		executeErr := storageTransformer.Execute(row)
 		if executeErr != nil {
 			if isKeyNotFound(executeErr) {
 				queueErr := watcher.Queue.Add(row)

pkg/datastore/postgres/repositories/block_repository.go

@@ -17,7 +17,6 @@
 package repositories
 
 import (
-	"context"
 	"database/sql"
 	"errors"
 	log "github.com/sirupsen/logrus"
@@ -122,7 +121,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
 
 func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) {
 	var blockId int64
-	tx, _ := blockRepository.database.BeginTx(context.Background(), nil)
+	tx, _ := blockRepository.database.Beginx()
 	err := tx.QueryRow(
 		`INSERT INTO blocks
                 (eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
@@ -145,7 +144,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 	return blockId, nil
 }
 
-func (blockRepository BlockRepository) createTransactions(tx *sql.Tx, blockId int64, transactions []core.Transaction) error {
+func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockId int64, transactions []core.Transaction) error {
 	for _, transaction := range transactions {
 		err := blockRepository.createTransaction(tx, blockId, transaction)
 		if err != nil {
@@ -165,7 +164,7 @@ func nullStringToZero(s string) string {
 	return s
 }
 
-func (blockRepository BlockRepository) createTransaction(tx *sql.Tx, blockId int64, transaction core.Transaction) error {
+func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.Transaction) error {
 	_, err := tx.Exec(
 		`INSERT INTO transactions
        (block_id, hash, nonce, tx_to, tx_from, gaslimit, gasprice, value, input_data)
@@ -198,7 +197,7 @@ func hasReceipt(transaction core.Transaction) bool {
 	return transaction.Receipt.TxHash != ""
 }
 
-func (blockRepository BlockRepository) createReceipt(tx *sql.Tx, blockId int64, receipt core.Receipt) (int, error) {
+func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) {
 	//Not currently persisting log bloom filters
 	var receiptId int
 	err := tx.QueryRow(
@@ -224,7 +223,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
 	return retrievedBlockHash, blockExists(retrievedBlockHash)
 }
 
-func (blockRepository BlockRepository) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error {
+func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error {
 	for _, tlog := range logs {
 		_, err := tx.Exec(
 			`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

pkg/datastore/postgres/repositories/logs_repository.go

@@ -17,7 +17,6 @@
 package repositories
 
 import (
-	"context"
 	"github.com/sirupsen/logrus"
 
 	"database/sql"
@@ -31,7 +30,7 @@ type LogRepository struct {
 }
 
 func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error {
-	tx, _ := logRepository.DB.BeginTx(context.Background(), nil)
+	tx, _ := logRepository.DB.Beginx()
 	for _, tlog := range lgs {
 		_, err := tx.Exec(
 			`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

pkg/datastore/postgres/repositories/receipt_repository.go

@@ -17,8 +17,8 @@
 package repositories
 
 import (
-	"context"
 	"database/sql"
+	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -31,7 +31,7 @@ type ReceiptRepository struct {
 }
 
 func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
-	tx, err := receiptRepository.DB.BeginTx(context.Background(), nil)
+	tx, err := receiptRepository.DB.Beginx()
 	if err != nil {
 		return err
 	}
@@ -53,7 +53,7 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64,
 	return nil
 }
 
-func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, error) {
+func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
 	var receiptId int64
 	err := tx.QueryRow(
 		`INSERT INTO receipts
@@ -68,7 +68,7 @@ func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, erro
 	return receiptId, err
 }
 
-func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
+func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
 	for _, log := range logs {
 		_, err := tx.Exec(
 			`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -84,7 +84,7 @@ func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
 }
 
 func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
-	tx, _ := receiptRepository.DB.BeginTx(context.Background(), nil)
+	tx, _ := receiptRepository.DB.Beginx()
 	var receiptId int64
 	err := tx.QueryRow(
 		`INSERT INTO receipts

pkg/fakes/data.go

@@ -24,7 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
@@ -49,5 +48,3 @@ func GetFakeHeader(blockNumber int64) core.Header {
 		Timestamp:   strconv.FormatInt(fakeTimestamp, 10),
 	}
 }
-
-var FakeHeaderTic = fakeTimestamp + constants.TTL

pkg/fakes/mock_filter_repository.go

@@ -0,0 +1,14 @@
+package fakes
+
+import "github.com/vulcanize/vulcanizedb/pkg/filters"
+
+type MockFilterRepository struct {
+}
+
+func (*MockFilterRepository) CreateFilter(filter filters.LogFilter) error {
+	return nil
+}
+
+func (*MockFilterRepository) GetFilter(name string) (filters.LogFilter, error) {
+	panic("implement me")
+}

pkg/fakes/mock_full_block_retriever.go

@@ -0,0 +1,15 @@
+package fakes
+
+type MockFullBlockRetriever struct {
+	FirstBlock      int64
+	FirstBlockErr   error
+	MostRecentBlock int64
+}
+
+func (retriever *MockFullBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
+	return retriever.FirstBlock, retriever.FirstBlockErr
+}
+
+func (retriever *MockFullBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
+	return retriever.MostRecentBlock, nil
+}

pkg/fakes/mock_light_block_retriever.go

@@ -0,0 +1,14 @@
+package fakes
+
+type MockLightBlockRetriever struct {
+	FirstBlock    int64
+	FirstBlockErr error
+}
+
+func (retriever *MockLightBlockRetriever) RetrieveFirstBlock() (int64, error) {
+	return retriever.FirstBlock, retriever.FirstBlockErr
+}
+
+func (retriever *MockLightBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
+	return 0, nil
+}

pkg/fakes/mock_light_header_repository.go

@@ -0,0 +1,42 @@
+package fakes
+
+import "github.com/vulcanize/vulcanizedb/pkg/core"
+
+type MockLightHeaderRepository struct {
+}
+
+func (*MockLightHeaderRepository) AddCheckColumn(id string) error {
+	return nil
+}
+
+func (*MockLightHeaderRepository) AddCheckColumns(ids []string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) CheckCache(key string) (interface{}, bool) {
+	panic("implement me")
+}

pkg/fakes/mock_parser.go

@@ -0,0 +1,40 @@
+package fakes
+
+import (
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
+)
+
+type MockParser struct {
+	AbiToReturn string
+	EventName   string
+	Event       types.Event
+}
+
+func (*MockParser) Parse(contractAddr string) error {
+	return nil
+}
+
+func (*MockParser) ParseAbiStr(abiStr string) error {
+	panic("implement me")
+}
+
+func (parser *MockParser) Abi() string {
+	return parser.AbiToReturn
+}
+
+func (*MockParser) ParsedAbi() abi.ABI {
+	return abi.ABI{}
+}
+
+func (*MockParser) GetMethods(wanted []string) []types.Method {
+	panic("implement me")
+}
+
+func (*MockParser) GetSelectMethods(wanted []string) []types.Method {
+	return []types.Method{}
+}
+
+func (parser *MockParser) GetEvents(wanted []string) map[string]types.Event {
+	return map[string]types.Event{parser.EventName: parser.Event}
+}

pkg/fakes/mock_poller.go

@@ -0,0 +1,24 @@
+package fakes
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
+)
+
+type MockPoller struct {
+	ContractName string
+}
+
+func (*MockPoller) PollContract(con contract.Contract) error {
+	panic("implement me")
+}
+
+func (*MockPoller) PollContractAt(con contract.Contract, blockNumber int64) error {
+	panic("implement me")
+}
+
+func (poller *MockPoller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
+	if p, ok := result.(*string); ok {
+		*p = poller.ContractName
+	}
+	return nil
+}

pkg/omni/full/converter/converter.go

@@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) {
 
 // Convert the given watched event log into a types.Log for the given event
 func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	values := make(map[string]interface{})
 	log := helpers.ConvertToLog(watchedEvent)
-	err := contract.UnpackLogIntoMap(values, event.Name, log)
+	err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 	if err != nil {
 		return nil, err
 	}

pkg/omni/full/transformer/transformer.go

@@ -34,7 +34,7 @@ import (
 )
 
 // Requires a fully synced vDB and a running eth node (or infura)
-type transformer struct {
+type Transformer struct {
 	// Database interfaces
 	datastore.FilterRepository       // Log filters repo; accepts filters generated by Contract.GenerateFilters()
 	datastore.WatchedEventRepository // Watched event log views, created by the log filters
@@ -76,8 +76,8 @@ type transformer struct {
 }
 
 // Transformer takes in config for blockchain, database, and network id
-func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transformer {
+func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer {
-	return &transformer{
+	return &Transformer{
 		Poller:                 poller.NewPoller(BC, DB, types.FullSync),
 		Parser:                 parser.NewParser(network),
 		BlockRetriever:         retriever.NewBlockRetriever(DB),
@@ -101,59 +101,59 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
 // Use this info to generate event filters
-func (t *transformer) Init() error {
+func (transformer *Transformer) Init() error {
-	for contractAddr, subset := range t.WatchedEvents {
+	for contractAddr, subset := range transformer.WatchedEvents {
 		// Get Abi
-		err := t.Parser.Parse(contractAddr)
+		err := transformer.Parser.Parse(contractAddr)
 		if err != nil {
 			return err
 		}
 
 		// Get first block and most recent block number in the header repo
-		firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
+		firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr)
 		if err != nil {
 			return err
 		}
-		lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
+		lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock()
 		if err != nil {
 			return err
 		}
 
 		// Set to specified range if it falls within the bounds
-		if firstBlock < t.ContractStart[contractAddr] {
+		if firstBlock < transformer.ContractStart[contractAddr] {
-			firstBlock = t.ContractStart[contractAddr]
+			firstBlock = transformer.ContractStart[contractAddr]
 		}
 
 		// Get contract name if it has one
 		var name = new(string)
-		t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock)
+		transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock)
 
 		// Remove any potential accidental duplicate inputs in arg filter values
 		eventArgs := map[string]bool{}
-		for _, arg := range t.EventArgs[contractAddr] {
+		for _, arg := range transformer.EventArgs[contractAddr] {
 			eventArgs[arg] = true
 		}
 		methodArgs := map[string]bool{}
-		for _, arg := range t.MethodArgs[contractAddr] {
+		for _, arg := range transformer.MethodArgs[contractAddr] {
 			methodArgs[arg] = true
 		}
 
 		// Aggregate info into contract object
 		info := contract.Contract{
 			Name:           *name,
-			Network:        t.Network,
+			Network:        transformer.Network,
 			Address:        contractAddr,
-			Abi:            t.Parser.Abi(),
+			Abi:            transformer.Parser.Abi(),
-			ParsedAbi:      t.Parser.ParsedAbi(),
+			ParsedAbi:      transformer.Parser.ParsedAbi(),
 			StartingBlock:  firstBlock,
 			LastBlock:      lastBlock,
-			Events:         t.Parser.GetEvents(subset),
+			Events:         transformer.Parser.GetEvents(subset),
-			Methods:        t.Parser.GetSelectMethods(t.WantedMethods[contractAddr]),
+			Methods:        transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]),
 			FilterArgs:     eventArgs,
 			MethodArgs:     methodArgs,
-			CreateAddrList: t.CreateAddrList[contractAddr],
+			CreateAddrList: transformer.CreateAddrList[contractAddr],
-			CreateHashList: t.CreateHashList[contractAddr],
+			CreateHashList: transformer.CreateHashList[contractAddr],
-			Piping:         t.Piping[contractAddr],
+			Piping:         transformer.Piping[contractAddr],
 		}.Init()
 
 		// Use info to create filters
@@ -164,14 +164,14 @@ func (t *transformer) Init() error {
 
 		// Iterate over filters and push them to the repo using filter repository interface
 		for _, filter := range info.Filters {
-			err = t.CreateFilter(filter)
+			err = transformer.FilterRepository.CreateFilter(filter)
 			if err != nil {
 				return err
 			}
 		}
 
 		// Store contract info for further processing
-		t.Contracts[contractAddr] = info
+		transformer.Contracts[contractAddr] = info
 	}
 
 	return nil
@@ -182,18 +182,18 @@ func (t *transformer) Init() error {
 // Uses converter to convert logs into custom log type
 // Persists converted logs into custuom postgres tables
 // Calls selected methods, using token holder address generated during event log conversion
-func (tr transformer) Execute() error {
+func (transformer Transformer) Execute() error {
-	if len(tr.Contracts) == 0 {
+	if len(transformer.Contracts) == 0 {
 		return errors.New("error: transformer has no initialized contracts to work with")
 	}
 	// Iterate through all internal contracts
-	for _, con := range tr.Contracts {
+	for _, con := range transformer.Contracts {
 		// Update converter with current contract
-		tr.Update(con)
+		transformer.Update(con)
 
 		// Iterate through contract filters and get watched event logs
 		for eventSig, filter := range con.Filters {
-			watchedEvents, err := tr.GetWatchedEvents(filter.Name)
+			watchedEvents, err := transformer.GetWatchedEvents(filter.Name)
 			if err != nil {
 				return err
 			}
@@ -201,7 +201,7 @@ func (tr transformer) Execute() error {
 			// Iterate over watched event logs
 			for _, we := range watchedEvents {
 				// Convert them to our custom log type
-				cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
+				cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig])
 				if err != nil {
 					return err
 				}
@@ -211,7 +211,7 @@ func (tr transformer) Execute() error {
 
 				// If log is not empty, immediately persist in repo
 				// Run this in seperate goroutine?
-				err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
+				err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
 				if err != nil {
 					return err
 				}
@@ -222,7 +222,7 @@ func (tr transformer) Execute() error {
 		// poller polls select contract methods
 		// and persists the results into custom pg tables
 		// Run this in seperate goroutine?
-		if err := tr.PollContract(*con); err != nil {
+		if err := transformer.PollContract(*con); err != nil {
 			return err
 		}
 	}
@@ -231,41 +231,41 @@ func (tr transformer) Execute() error {
 }
 
 // Used to set which contract addresses and which of their events to watch
-func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) {
-	tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
+	transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to watch events for
-func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
-	tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
+	transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set which contract addresses and which of their methods to call
-func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) {
-	tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
+	transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to poll methods on
-func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
-	tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
+	transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set the block range to watch for a given address
-func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
+func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) {
-	tr.ContractStart[strings.ToLower(contractAddr)] = start
+	transformer.ContractStart[strings.ToLower(contractAddr)] = start
 }
 
 // Used to set whether or not to persist an account address list
-func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
+func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) {
-	tr.CreateAddrList[strings.ToLower(contractAddr)] = on
+	transformer.CreateAddrList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to set whether or not to persist an hash list
-func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
+func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) {
-	tr.CreateHashList[strings.ToLower(contractAddr)] = on
+	transformer.CreateHashList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to turn method piping on for a contract
-func (tr *transformer) SetPiping(contractAddr string, on bool) {
+func (transformer *Transformer) SetPiping(contractAddr string, on bool) {
-	tr.Piping[strings.ToLower(contractAddr)] = on
+	transformer.Piping[strings.ToLower(contractAddr)] = on
 }

pkg/omni/full/transformer/transformer_test.go

@@ -17,353 +17,160 @@
 package transformer_test
 
 import (
-	"fmt"
 	"math/rand"
-	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
 )
 
 var _ = Describe("Transformer", func() {
-	var db *postgres.DB
+	var fakeAddress = "0x1234567890abcdef"
-	var err error
-	var blockChain core.BlockChain
-	var blockRepository repositories.BlockRepository
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
 	rand.Seed(time.Now().UnixNano())
 
-	BeforeEach(func() {
-		db, blockChain = test_helpers.SetupDBandBC()
-		blockRepository = *repositories.NewBlockRepository(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
 	Describe("SetEvents", func() {
 		It("Sets which events to watch from the given contract address", func() {
 			watchedEvents := []string{"Transfer", "Mint"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEvents(constants.TusdContractAddress, watchedEvents)
+			t.SetEvents(fakeAddress, watchedEvents)
-			Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
+			Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
 		})
 	})
 
 	Describe("SetEventAddrs", func() {
 		It("Sets which account addresses to watch events for", func() {
 			eventAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
+			t.SetEventArgs(fakeAddress, eventAddrs)
-			Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
+			Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
 		})
 	})
 
 	Describe("SetMethods", func() {
 		It("Sets which methods to poll at the given contract address", func() {
 			watchedMethods := []string{"balanceOf", "totalSupply"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethods(constants.TusdContractAddress, watchedMethods)
+			t.SetMethods(fakeAddress, watchedMethods)
-			Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
+			Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
 		})
 	})
 
 	Describe("SetMethodAddrs", func() {
 		It("Sets which account addresses to poll methods against", func() {
 			methodAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
+			t.SetMethodArgs(fakeAddress, methodAddrs)
-			Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
+			Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
 		})
 	})
 
 	Describe("SetStartingBlock", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetStartingBlock(constants.TusdContractAddress, 11)
+			t.SetStartingBlock(fakeAddress, 11)
-			Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
+			Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
 		})
 	})
 
 	Describe("SetCreateAddrList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateAddrList(constants.TusdContractAddress, true)
+			t.SetCreateAddrList(fakeAddress, true)
-			Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("SetCreateHashList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateHashList(constants.TusdContractAddress, true)
+			t.SetCreateHashList(fakeAddress, true)
-			Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			blockRetriever := &fakes.MockFullBlockRetriever{}
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+			firstBlock := int64(1)
-			t := transformer.NewTransformer("", blockChain, db)
+			mostRecentBlock := int64(2)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			blockRetriever.FirstBlock = firstBlock
-			err = t.Init()
+			blockRetriever.MostRecentBlock = mostRecentBlock
+
+			parsr := &fakes.MockParser{}
+			fakeAbi := "fake_abi"
+			eventName := "Transfer"
+			event := types.Event{}
+			parsr.AbiToReturn = fakeAbi
+			parsr.EventName = eventName
+			parsr.Event = event
+
+			pollr := &fakes.MockPoller{}
+			fakeContractName := "fake_contract_name"
+			pollr.ContractName = fakeContractName
+
+			t := getTransformer(blockRetriever, parsr, pollr)
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			c, ok := t.Contracts[tusdAddr]
+			c, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(true))
 
-			Expect(c.StartingBlock).To(Equal(int64(6194633)))
+			Expect(c.StartingBlock).To(Equal(firstBlock))
-			Expect(c.LastBlock).To(Equal(int64(6194634)))
+			Expect(c.LastBlock).To(Equal(mostRecentBlock))
-			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(tusdAddr))
+			Expect(c.Address).To(Equal(fakeAddress))
 		})
 
 		It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			blockRetriever := &fakes.MockFullBlockRetriever{}
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			blockRetriever.FirstBlockErr = fakes.FakeError
-			err = t.Init()
+			t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
 		})
 
 		It("Does nothing if watched events are unset", func() {
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+
-			t := transformer.NewTransformer("", blockChain, db)
+			err := t.Init()
-			err = t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			_, ok := t.Contracts[tusdAddr]
+			_, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(false))
 		})
 	})
-
-	Describe("Execute", func() {
-		BeforeEach(func() {
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.TransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
-
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
-			Expect(log.Block).To(Equal(int64(6194634)))
-			Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
-			Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
-			Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
-		})
-
-		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.BalanceOf{}
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("0"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("0"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("Fails if initialization has not been done", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-
-			err = t.Execute()
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("Execute- against ENS registry contract", func() {
-		BeforeEach(func() {
-			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
-			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.NewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
-
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
-			Expect(log.Block).To(Equal(int64(6194635)))
-			Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
-			Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
-			Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
-		})
-
-		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedHashes)).To(Equal(3))
-
-			b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("It does not perist events if they do not pass the emitted arg filter", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
-
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("If a method arg filter is applied, only those arguments are used in polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-	})
 })
+
+func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
+	return transformer.Transformer{
+		FilterRepository: &fakes.MockFilterRepository{},
+		Parser:           parsr,
+		BlockRetriever:   blockRetriever,
+		Poller:           pollr,
+		Contracts:        map[string]*contract.Contract{},
+		WatchedEvents:    map[string][]string{},
+		WantedMethods:    map[string][]string{},
+		ContractStart:    map[string]int64{},
+		EventArgs:        map[string][]string{},
+		MethodArgs:       map[string][]string{},
+		CreateAddrList:   map[string]bool{},
+		CreateHashList:   map[string]bool{},
+	}
+}

pkg/omni/light/converter/converter.go

@@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) {
 
 // Convert the given watched event log into a types.Log for the given event
 func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	returnLogs := make([]types.Log, 0, len(logs))
 	for _, log := range logs {
 		values := make(map[string]interface{})
@@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 			values[field.Name] = i
 		}
 
-		err := contract.UnpackLogIntoMap(values, event.Name, log)
+		err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 		if err != nil {
 			return nil, err
 		}
@@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 
 // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
 func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	eventsToLogs := make(map[string][]types.Log)
 	for _, event := range events {
 		eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
 			// If the log is of this event type, process it as such
 			if event.Sig() == log.Topics[0] {
 				values := make(map[string]interface{})
-				err := contract.UnpackLogIntoMap(values, event.Name, log)
+				err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 				if err != nil {
 					return nil, err
 				}

pkg/omni/light/repository/header_repository.go

@@ -17,8 +17,8 @@
 package repository
 
 import (
-	"database/sql"
 	"fmt"
+	"github.com/jmoiron/sqlx"
 
 	"github.com/hashicorp/golang-lru"
 
@@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
 }
 
 func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
-	tx, err := r.db.Begin()
+	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
 	}
@@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
 	return r.columns.Get(key)
 }
 
-func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error {
+func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
 	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
 		VALUES ($1, $2) 
 		ON CONFLICT (header_id) DO

pkg/omni/light/retriever/retriever_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestRetriever(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light BLock Number Retriever Suite Test")
+	RunSpecs(t, "Light Block Number Retriever Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/omni/light/transformer/transformer.go

@@ -37,7 +37,7 @@ import (
 )
 
 // Requires a light synced vDB (headers) and a running eth node (or infura)
-type transformer struct {
+type Transformer struct {
 	// Database interfaces
 	srep.EventRepository        // Holds transformed watched event log data
 	repository.HeaderRepository // Interface for interaction with header repositories
@@ -93,9 +93,9 @@ type transformer struct {
 // 4. Execute
 
 // Transformer takes in config for blockchain, database, and network id
-func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transformer {
+func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer {
 
-	return &transformer{
+	return &Transformer{
 		Poller:           poller.NewPoller(bc, db, types.LightSync),
 		Fetcher:          fetcher.NewFetcher(bc),
 		Parser:           parser.NewParser(network),
@@ -120,7 +120,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
 // Use this info to generate event filters
-func (tr *transformer) Init() error {
+func (tr *Transformer) Init() error {
 	// Initialize internally configured transformer settings
 	tr.contractAddresses = make([]string, 0)       // Holds all contract addresses, for batch fetching of logs
 	tr.sortedEventIds = make(map[string][]string)  // Map to sort event column ids by contract, for post fetch processing and persisting of logs
@@ -154,7 +154,7 @@ func (tr *transformer) Init() error {
 
 		// Get contract name if it has one
 		var name = new(string)
-		tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
+		tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock)
 
 		// Remove any potential accidental duplicate inputs in arg filter values
 		eventArgs := map[string]bool{}
@@ -221,7 +221,7 @@ func (tr *transformer) Init() error {
 	return nil
 }
 
-func (tr *transformer) Execute() error {
+func (tr *Transformer) Execute() error {
 	if len(tr.Contracts) == 0 {
 		return errors.New("error: transformer has no initialized contracts")
 	}
@@ -311,7 +311,7 @@ func (tr *transformer) Execute() error {
 }
 
 // Used to poll contract methods at a given header
-func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
+func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
 	for _, con := range tr.Contracts {
 		// Skip method polling processes if no methods are specified
 		// Also don't try to poll methods below this contract's specified starting block
@@ -336,41 +336,41 @@ func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[str
 }
 
 // Used to set which contract addresses and which of their events to watch
-func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) {
 	tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to watch events for
-func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
 	tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set which contract addresses and which of their methods to call
-func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) {
 	tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to poll methods on
-func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
 	tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set the block range to watch for a given address
-func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
+func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) {
 	tr.ContractStart[strings.ToLower(contractAddr)] = start
 }
 
 // Used to set whether or not to persist an account address list
-func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
+func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) {
 	tr.CreateAddrList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to set whether or not to persist an hash list
-func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
+func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) {
 	tr.CreateHashList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to turn method piping on for a contract
-func (tr *transformer) SetPiping(contractAddr string, on bool) {
+func (tr *Transformer) SetPiping(contractAddr string, on bool) {
 	tr.Piping[strings.ToLower(contractAddr)] = on
 }

pkg/omni/light/transformer/transformer_test.go

@@ -17,491 +17,150 @@
 package transformer_test
 
 import (
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
 )
 
 var _ = Describe("Transformer", func() {
-	var db *postgres.DB
+	var fakeAddress = "0x1234567890abcdef"
-	var err error
-	var blockChain core.BlockChain
-	var headerRepository repositories.HeaderRepository
-	var headerID, headerID2 int64
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
-
-	BeforeEach(func() {
-		db, blockChain = test_helpers.SetupDBandBC()
-		headerRepository = repositories.NewHeaderRepository(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
 
 	Describe("SetEvents", func() {
 		It("Sets which events to watch from the given contract address", func() {
 			watchedEvents := []string{"Transfer", "Mint"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEvents(constants.TusdContractAddress, watchedEvents)
+			t.SetEvents(fakeAddress, watchedEvents)
-			Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
+			Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
 		})
 	})
 
 	Describe("SetEventAddrs", func() {
 		It("Sets which account addresses to watch events for", func() {
 			eventAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
+			t.SetEventArgs(fakeAddress, eventAddrs)
-			Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
+			Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
 		})
 	})
 
 	Describe("SetMethods", func() {
 		It("Sets which methods to poll at the given contract address", func() {
 			watchedMethods := []string{"balanceOf", "totalSupply"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethods(constants.TusdContractAddress, watchedMethods)
+			t.SetMethods(fakeAddress, watchedMethods)
-			Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
+			Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
 		})
 	})
 
 	Describe("SetMethodAddrs", func() {
 		It("Sets which account addresses to poll methods against", func() {
 			methodAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
+			t.SetMethodArgs(fakeAddress, methodAddrs)
-			Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
+			Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
 		})
 	})
 
 	Describe("SetStartingBlock", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetStartingBlock(constants.TusdContractAddress, 11)
+			t.SetStartingBlock(fakeAddress, 11)
-			Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
+			Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
 		})
 	})
 
 	Describe("SetCreateAddrList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateAddrList(constants.TusdContractAddress, true)
+			t.SetCreateAddrList(fakeAddress, true)
-			Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("SetCreateHashList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateHashList(constants.TusdContractAddress, true)
+			t.SetCreateHashList(fakeAddress, true)
-			Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			blockRetriever := &fakes.MockLightBlockRetriever{}
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			firstBlock := int64(1)
-			t := transformer.NewTransformer("", blockChain, db)
+			blockRetriever.FirstBlock = firstBlock
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+
-			err = t.Init()
+			parsr := &fakes.MockParser{}
+			fakeAbi := "fake_abi"
+			parsr.AbiToReturn = fakeAbi
+
+			pollr := &fakes.MockPoller{}
+			fakeContractName := "fake_contract_name"
+			pollr.ContractName = fakeContractName
+
+			t := getFakeTransformer(blockRetriever, parsr, pollr)
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			c, ok := t.Contracts[tusdAddr]
+			c, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(true))
 
-			Expect(c.StartingBlock).To(Equal(int64(6194632)))
+			Expect(c.StartingBlock).To(Equal(firstBlock))
 			Expect(c.LastBlock).To(Equal(int64(-1)))
-			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(tusdAddr))
+			Expect(c.Address).To(Equal(fakeAddress))
 		})
 
 		It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			blockRetriever := &fakes.MockLightBlockRetriever{}
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			blockRetriever.FirstBlockErr = fakes.FakeError
-			err = t.Init()
+			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
 		})
 
 		It("Does nothing if watched events are unset", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+
-			t := transformer.NewTransformer("", blockChain, db)
+			err := t.Init()
-			err = t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			_, ok := t.Contracts[tusdAddr]
+			_, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(false))
 		})
 	})
 
-	Describe("Execute- against TrueUSD contract", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6791668)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6791669)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6791670)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.HeaderID).To(Equal(headerID))
-			Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
-			Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
-			Expect(log.Value).To(Equal("9998940000000000000000"))
-		})
-
-		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			c, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedAddrs)).To(Equal(4))
-			Expect(len(c.EmittedHashes)).To(Equal(0))
-
-			b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("55849938025000000000000"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("Fails if initialization has not been done", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-			err = t.Execute()
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("Execute- against ENS registry contract", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6885695)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6885696)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6885697)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.HeaderID).To(Equal(headerID))
-			Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
-			Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
-			Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-		})
-
-		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			c, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedHashes)).To(Equal(2))
-			Expect(len(c.EmittedAddrs)).To(Equal(0))
-
-			b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given method using list of collected hashes", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("It does not persist events if they do not pass the emitted arg filter", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("If a method arg filter is applied, only those arguments are used in polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("Execute- against both ENS and TrueUSD", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6791668)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6791669)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6791670)
-			Expect(err).ToNot(HaveOccurred())
-			header4, err := blockChain.GetHeaderByNumber(6885695)
-			Expect(err).ToNot(HaveOccurred())
-			header5, err := blockChain.GetHeaderByNumber(6885696)
-			Expect(err).ToNot(HaveOccurred())
-			header6, err := blockChain.GetHeaderByNumber(6885697)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-			headerRepository.CreateOrUpdateHeader(header4)
-			headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header6)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			newOwnerLog := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
-			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
-			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
-			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-
-			transferLog := test_helpers.LightTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(transferLog.HeaderID).To(Equal(headerID))
-			Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
-			Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
-			Expect(transferLog.Value).To(Equal("9998940000000000000000"))
-		})
-
-		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			ens, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-			tusd, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(ens.EmittedHashes)).To(Equal(2))
-			Expect(len(ens.EmittedAddrs)).To(Equal(0))
-			Expect(len(tusd.EmittedAddrs)).To(Equal(4))
-			Expect(len(tusd.EmittedHashes)).To(Equal(0))
-
-			b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-		})
-
-		It("Polls given methods for each contract, using list of collected values", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			owner := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(owner.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(owner.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).To(HaveOccurred())
-
-			bal := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal.Balance).To(Equal("55849938025000000000000"))
-			Expect(bal.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
-			Expect(err).To(HaveOccurred())
-		})
-	})
 })
+
+func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
+	return transformer.Transformer{
+		Parser:           parsr,
+		BlockRetriever:   blockRetriever,
+		Poller:           pollr,
+		HeaderRepository: &fakes.MockLightHeaderRepository{},
+		Contracts:        map[string]*contract.Contract{},
+		WatchedEvents:    map[string][]string{},
+		WantedMethods:    map[string][]string{},
+		ContractStart:    map[string]int64{},
+		EventArgs:        map[string][]string{},
+		MethodArgs:       map[string][]string{},
+		CreateAddrList:   map[string]bool{},
+		CreateHashList:   map[string]bool{},
+	}
+}

pkg/omni/shared/constants/constants.go

@@ -72,6 +72,7 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
 var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b"
 var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5"
 
+// TODO: Consider whether these should be moved to plugins
 // Contract Owner
 var DaiContractOwner = "0x0000000000000000000000000000000000000000"
 var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f"

pkg/omni/shared/helpers/test_helpers/database.go

@@ -106,6 +106,7 @@ type Owner struct {
 	Address   string `db:"returned"`
 }
 
+// TODO: consider whether this should be moved to libraries/shared
 func SetupBC() core.BlockChain {
 	infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
 	rawRpcClient, err := rpc.Dial(infuraIPC)
@@ -113,9 +114,9 @@ func SetupBC() core.BlockChain {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	blockChainNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
 
 	return blockChain
 }
@@ -127,9 +128,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	blockChainNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
 
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",
@@ -188,6 +189,7 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract
 	}.Init()
 }
 
+// TODO: consider whether this can be moved to plugin or libraries/shared
 func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) {
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",
@@ -236,7 +238,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
 }
 
 func TearDown(db *postgres.DB) {
-	tx, err := db.Begin()
+	tx, err := db.Beginx()
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DELETE FROM blocks`)

pkg/omni/shared/repository/event_repository.go

@@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
 
 // Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
 func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Begin()
+	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
 	}
@@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
 
 // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
 func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Begin()
+	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
 	}

pkg/omni/shared/repository/method_repository.go

@@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ
 
 // Creates a custom postgres command to persist logs for the given event
 func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
-	tx, err := r.DB.Begin()
+	tx, err := r.DB.Beginx()
 	if err != nil {
 		return err
 	}

pkg/plugin/test_helpers/database.go

@@ -31,6 +31,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
+// TODO: consider whether this should be moved to libraries/shared
 func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	infuraIPC := "http://kovan0.vulcanize.io:8545"
 	rawRpcClient, err := rpc.Dial(infuraIPC)
@@ -38,9 +39,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	blockChainNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
 
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",

scripts/start_blockchain

@@ -1,2 +0,0 @@
-#!/bin/bash
-geth --fast --cache=1024

scripts/start_test_chain.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-MNEMONIC_PHRASE="whisper ordinary mystery awesome wood fox auction february blind volcano spare soft"
-PORT=7545
-DATABASE_PATH=test_data/test_chain/
-echo Starting ganache chain on port $PORT...
-
-ganache-cli --port $PORT \
-            --db $DATABASE_PATH \
-            2>&1 > ganache-output.log &

scripts/stop_test_chain.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-echo "Stopping ganache chain on port 7545"
-ps -ef | grep ganache | grep -v grep | awk '{print $2}' | xargs kill