Merge branch 'staging' into unpack_update

.private_blockchain_password

@@ -1 +0,0 @@
-1234

.travis.yml

@@ -6,26 +6,20 @@ services:
   - postgresql
 addons:
   postgresql: "9.6"
-  ssh_known_hosts:
-    - 147.75.96.51
 
 go_import_path: github.com/vulcanize/vulcanizedb
 
 before_install:
   # ginkgo golint dep goose
-  - echo -e "Host github.com\n\tHostName github.com\n\tUser git\n\tIdentityFile ~/.ssh/id_rsa\n" >> ~/.ssh/config
   - make installtools
   - bash ./scripts/install-postgres-10.sh
-  - npm install -g ganache-cli
   - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
   - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
   - sudo apt-get update && sudo apt-get install yarn
 
 before_script:
-  - go get -u github.com/pressly/sup/cmd/sup
   - sudo -u postgres createdb vulcanize_private
   - make migrate NAME=vulcanize_private
-  - bash ./scripts/start_test_chain.sh
   - cd postgraphile && yarn
 
 script:
@@ -36,7 +30,3 @@ script:
 
 notifications:
   email: false
-
-after_script:
-  - bash ./scripts/stop_test_chain.sh
-  - bash ./bin/deploy.sh

README.md

@@ -31,7 +31,6 @@ Using Vulcanize for the first time requires several steps be done in order to al
 In order to fetch the project codebase for local use or modification, install it to your `GOPATH` via:
 
 `go get github.com/vulcanize/vulcanizedb`
-`go get gopkg.in/DataDog/dd-trace-go.v1/ddtrace`
 
 Once fetched, dependencies can be installed via `go get` or (the preferred method) at specific versions via `golang/dep`, the prototype golang pakcage manager. Installation instructions are [here](https://golang.github.io/dep/docs/installation.html).
 
@@ -252,8 +251,8 @@ Storage Transformers
    * [Example](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/factories/storage/EXAMPLE.md)   
     
 Event Transformers   
-   * Guide
+   * [Guide](https://github.com/vulcanize/maker-vulcanizedb/blob/event_docs/libraries/shared/factories/README.md)
-   * Example
+   * [Example](https://github.com/vulcanize/ens_transformers/tree/working)
    
 #### composeAndExecute configuration
 A .toml config file is specified when executing the command:
@@ -312,9 +311,9 @@ The config provides information for composing a set of transformers:
     - `path` is the relative path from `repository` to the transformer's `TransformerInitializer` directory (initializer package).
         - Transformer repositories need to be cloned into the user's $GOPATH (`go get`)
     - `type` is the type of the transformer; indicating which type of watcher it works with (for now, there are only two options: `eth_event` and `eth_storage`)
-        - `eth_storage` indicates the transformer works with the [storage watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go)
+        - `eth_storage` indicates the transformer works with the [storage watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/storage_watcher.go)
          that fetches state and storage diffs from an ETH node (instead of, for example, from IPFS)
-        - `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go)
+        - `eth_event` indicates the transformer works with the [event watcher](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go)
          that fetches event logs from an ETH node
     - `migrations` is the relative path from `repository` to the db migrations directory for the transformer
 - Note: If any of the imported transformers need additional config variables those need to be included as well
@@ -355,14 +354,16 @@ func (e exporter) Export() []interface1.TransformerInitializer, []interface1.Sto
 #### Preparing transformer(s) to work as pluggins for composeAndExecute
 To plug in an external transformer we need to:
 
-* Create a [package](https://github.com/vulcanize/mcd_transformers/blob/staging/transformers/bite/initializer/initializer.go) 
+* Create a [package](https://github.com/vulcanize/ens_transformers/blob/working/transformers/registry/new_owner/initializer/initializer.go)
-that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/event_transformer.go#L33)
+that exports a variable `TransformerInitializer` or `StorageTransformerInitializer` that are of type [TransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/event_transformer.go#L33)
-or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/transformer/storage_transformer.go#L31), respectively   
+or [StorageTransformerInitializer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/storage_transformer.go#L31), respectively
-* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/event_watcher.go#L83)
+* Design the transformers to work in the context of their [event](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go#L83)
-or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/compose_and_execute/libraries/shared/watcher/storage_watcher.go#L53) watchers
+or [storage](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/storage_watcher.go#L58) watcher execution modes
 * Create db migrations to run against vulcanizeDB so that we can store the transformer output
-    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
     * Do not `goose fix` the transformer migrations   
+    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
+    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
+    as they are [here](https://github.com/vulcanize/vulcanizedb/tree/master/db/migrations)
 
 To update a plugin repository with changes to the core vulcanizedb repository, replace the vulcanizedb vendored in the plugin repo (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb`)
 with the newly updated version

Supfile

@@ -1,64 +0,0 @@
----
-version: 0.5
-
-env:
-  VDB_PATH: /root/go_projects/src/github.com/vulcanize/vulcanizedb
-  VDB_PG_USER: vulcanize
-  VDB_PG_PW: vulcanize
-
-networks:
-  staging:
-    hosts:
-      - root@147.75.96.51
-  prod:
-    hosts:
-      - root@147.75.197.13
-
-targets:
-  deploy:
-    - remove
-    - transfer
-    - buildPostgraphile
-    - buildVDB
-    - migrate
-    - lightSync
-    - postgraphile
-
-commands:
-  remove:
-    desc: remove old vulcanizedb
-    run: rm -rf $VDB_PATH && rm -rf /usr/local/vulcanizedb && mkdir -p $VDB_PATH
-  transfer:
-    desc: transfer repo to remote server
-    upload:
-      - src: .
-        dst: $VDB_PATH
-  migrate:
-    desc: run migration
-    run: >
-      cd $VDB_PATH &&
-      make installtools &&
-      cd db/migrations &&
-      /root/go_projects/bin/goose postgres "postgresql://$(VDB_PG_USER):$(VDB_PG_PW)@127.0.0.1:5432/vulcanize_public?sslmode=disable" up
-  buildPostgraphile:
-    desc: build postgraphile app
-    run: >
-      cd $VDB_PATH/postgraphile &&
-      yarn && tsc
-  buildVDB:
-    desc: build vulcanizedb
-    run: >
-      cd $VDB_PATH &&
-      GOPATH=$HOME/go_projects go get &&
-      GOPATH=$HOME/go_projects go build &&
-      cp -r . /usr/local/vulcanizedb
-  lightSync:
-    desc: start vdb light sync
-    run: >
-      systemctl daemon-reload &&
-      sudo systemctl restart vulcanizedb_light_sync.service &&
-      sudo systemctl restart vulcanizedb_log_sync.service &&
-      sudo systemctl restart vulcanizedb_recheck_sync.service
-  postgraphile:
-    desc: start postgraphile
-    run: systemctl daemon-reload && sudo systemctl restart postgraphile.service

bin/deploy.sh

@@ -1,5 +0,0 @@
-if [ $TRAVIS_BRANCH == 'staging' ]; then
-  sup --debug staging deploy
-elif [ $TRAVIS_BRANCH == 'master' ]; then
-  sup --debug prod deploy
-fi

cmd/root.go

@@ -94,7 +94,6 @@ func init() {
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
-	rootCmd.PersistentFlags().String("datadog-name", "vulcanize-test", "datadog service name")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 
@@ -105,7 +104,6 @@ func init() {
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
-	viper.BindPFlag("datadog.name", rootCmd.PersistentFlags().Lookup("datadog-name"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 }

db/migrations/00042_create_token_supply_table.sql

@@ -1,14 +0,0 @@
--- +goose Up
-CREATE TABLE token_supply (
-  id            SERIAL,
-  block_id      INTEGER NOT NULL,
-  supply        DECIMAL NOT NULL,
-  token_address CHARACTER VARYING(66) NOT NULL,
-  CONSTRAINT blocks_fk FOREIGN KEY (block_id)
-  REFERENCES blocks (id)
-  ON DELETE CASCADE
-);
-
-
--- +goose Down
-DROP TABLE token_supply;

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.5
+-- Dumped from database version 10.6
--- Dumped by pg_dump version 10.5
+-- Dumped by pg_dump version 10.6
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -368,38 +368,6 @@ CREATE SEQUENCE public.receipts_id_seq
 ALTER SEQUENCE public.receipts_id_seq OWNED BY public.receipts.id;
 
 
---
--- Name: token_supply; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.token_supply (
-    id integer NOT NULL,
-    block_id integer NOT NULL,
-    supply numeric NOT NULL,
-    token_address character varying(66) NOT NULL
-);
-
-
---
--- Name: token_supply_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.token_supply_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: token_supply_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.token_supply_id_seq OWNED BY public.token_supply.id;
-
-
 --
 -- Name: transactions; Type: TABLE; Schema: public; Owner: -
 --
@@ -555,13 +523,6 @@ ALTER TABLE ONLY public.queued_storage ALTER COLUMN id SET DEFAULT nextval('publ
 ALTER TABLE ONLY public.receipts ALTER COLUMN id SET DEFAULT nextval('public.receipts_id_seq'::regclass);
 
 
---
--- Name: token_supply id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.token_supply ALTER COLUMN id SET DEFAULT nextval('public.token_supply_id_seq'::regclass);
-
-
 --
 -- Name: transactions id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -754,14 +715,6 @@ ALTER TABLE ONLY public.receipts
     ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
 
 
---
--- Name: token_supply blocks_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.token_supply
-    ADD CONSTRAINT blocks_fk FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
-
-
 --
 -- Name: checked_headers checked_headers_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

integration_test/omni_full_transformer_test.go

@@ -0,0 +1,275 @@
+package integration
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+)
+
+var _ = Describe("Omni full transformer", func() {
+	var db *postgres.DB
+	var err error
+	var blockChain core.BlockChain
+	var blockRepository repositories.BlockRepository
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+	rand.Seed(time.Now().UnixNano())
+
+	BeforeEach(func() {
+		db, blockChain = test_helpers.SetupDBandBC()
+		blockRepository = *repositories.NewBlockRepository(db)
+	})
+
+	AfterEach(func() {
+		test_helpers.TearDown(db)
+	})
+
+	Describe("Init", func() {
+		It("Initializes transformer's contract objects", func() {
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+
+			Expect(c.StartingBlock).To(Equal(int64(6194633)))
+			Expect(c.LastBlock).To(Equal(int64(6194634)))
+			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Address).To(Equal(tusdAddr))
+		})
+	})
+
+	Describe("Execute", func() {
+		BeforeEach(func() {
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.TransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
+
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
+			Expect(log.Block).To(Equal(int64(6194634)))
+			Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
+			Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
+			Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
+		})
+
+		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[""]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.BalanceOf{}
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Balance).To(Equal("0"))
+			Expect(res.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Balance).To(Equal("0"))
+			Expect(res.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Fails if initialization has not been done", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+
+			err = t.Execute()
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Execute- against ENS registry contract", func() {
+		BeforeEach(func() {
+			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
+			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.NewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
+
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
+			Expect(log.Block).To(Equal(int64(6194635)))
+			Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
+			Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
+			Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
+		})
+
+		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[ensAddr]
+			Expect(ok).To(Equal(true))
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedHashes)).To(Equal(3))
+
+			b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("It does not perist events if they do not pass the emitted arg filter", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
+
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("If a method arg filter is applied, only those arguments are used in polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

integration_test/omni_light_transformer_test.go

@@ -0,0 +1,413 @@
+package integration
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+)
+
+var _ = Describe("Omnit light transformer", func() {
+	var db *postgres.DB
+	var err error
+	var blockChain core.BlockChain
+	var headerRepository repositories.HeaderRepository
+	var headerID, headerID2 int64
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+
+	BeforeEach(func() {
+		db, blockChain = test_helpers.SetupDBandBC()
+		headerRepository = repositories.NewHeaderRepository(db)
+	})
+
+	AfterEach(func() {
+		test_helpers.TearDown(db)
+	})
+
+	Describe("Init", func() {
+		It("Initializes transformer's contract objects", func() {
+			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+
+			Expect(c.StartingBlock).To(Equal(int64(6194632)))
+			Expect(c.LastBlock).To(Equal(int64(-1)))
+			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Address).To(Equal(tusdAddr))
+		})
+	})
+
+	Describe("Execute- against TrueUSD contract", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6791668)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6791669)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6791670)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.HeaderID).To(Equal(headerID))
+			Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
+			Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
+			Expect(log.Value).To(Equal("9998940000000000000000"))
+		})
+
+		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			c, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedAddrs)).To(Equal(4))
+			Expect(len(c.EmittedHashes)).To(Equal(0))
+
+			b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[""]
+			Expect(ok).To(Equal(false))
+
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given methods using generated token holder address", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.BalanceOf{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Balance).To(Equal("55849938025000000000000"))
+			Expect(res.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Fails if initialization has not been done", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Execute()
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Execute- against ENS registry contract", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6885695)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6885696)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6885697)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(log.HeaderID).To(Equal(headerID))
+			Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
+			Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
+			Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+		})
+
+		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			c, ok := t.Contracts[ensAddr]
+			Expect(ok).To(Equal(true))
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(c.EmittedHashes)).To(Equal(2))
+			Expect(len(c.EmittedAddrs)).To(Equal(0))
+
+			b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
+			_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
+			Expect(ok).To(Equal(false))
+		})
+
+		It("Polls given method using list of collected hashes", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("It does not persist events if they do not pass the emitted arg filter", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			log := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("If a method arg filter is applied, only those arguments are used in polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			res := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(res.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Execute- against both ENS and TrueUSD", func() {
+		BeforeEach(func() {
+			header1, err := blockChain.GetHeaderByNumber(6791668)
+			Expect(err).ToNot(HaveOccurred())
+			header2, err := blockChain.GetHeaderByNumber(6791669)
+			Expect(err).ToNot(HaveOccurred())
+			header3, err := blockChain.GetHeaderByNumber(6791670)
+			Expect(err).ToNot(HaveOccurred())
+			header4, err := blockChain.GetHeaderByNumber(6885695)
+			Expect(err).ToNot(HaveOccurred())
+			header5, err := blockChain.GetHeaderByNumber(6885696)
+			Expect(err).ToNot(HaveOccurred())
+			header6, err := blockChain.GetHeaderByNumber(6885697)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header1)
+			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header3)
+			headerRepository.CreateOrUpdateHeader(header4)
+			headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
+			Expect(err).ToNot(HaveOccurred())
+			headerRepository.CreateOrUpdateHeader(header6)
+		})
+
+		It("Transforms watched contract data into custom repositories", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, nil)
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, nil)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			newOwnerLog := test_helpers.LightNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
+			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
+			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
+			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+
+			transferLog := test_helpers.LightTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			Expect(err).ToNot(HaveOccurred())
+			// We don't know vulcID, so compare individual fields instead of complete structures
+			Expect(transferLog.HeaderID).To(Equal(headerID))
+			Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
+			Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
+			Expect(transferLog.Value).To(Equal("9998940000000000000000"))
+		})
+
+		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			ens, ok := t.Contracts[ensAddr]
+			Expect(ok).To(Equal(true))
+			tusd, ok := t.Contracts[tusdAddr]
+			Expect(ok).To(Equal(true))
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(ens.EmittedHashes)).To(Equal(2))
+			Expect(len(ens.EmittedAddrs)).To(Equal(0))
+			Expect(len(tusd.EmittedAddrs)).To(Equal(4))
+			Expect(len(tusd.EmittedHashes)).To(Equal(0))
+
+			b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+		})
+
+		It("Polls given methods for each contract, using list of collected values", func() {
+			t := transformer.NewTransformer("", blockChain, db)
+			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
+			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
+			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+
+			owner := test_helpers.Owner{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
+			Expect(owner.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
+			Expect(owner.TokenName).To(Equal(""))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			Expect(err).To(HaveOccurred())
+
+			bal := test_helpers.BalanceOf{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal.Balance).To(Equal("55849938025000000000000"))
+			Expect(bal.TokenName).To(Equal("TrueUSD"))
+
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

libraries/shared/README.md

@@ -0,0 +1,8 @@
+# Shared Tools
+
+## Description
+Code that is useful for or used by plugins written on top of VulcanizeDB.
+
+## Note
+Much code in this directory may not be used outside of the tests, but don't delete it - it could be used by a plugin.
+Renaming and/or deleting functions in this namespace requires a version bump to avoid breaking plugins.

libraries/shared/chunker/log_chunker_test.go

@@ -58,9 +58,9 @@ var _ = Describe("Log chunker", func() {
 	Describe("initialisation", func() {
 		It("creates lookup maps correctly", func() {
 			Expect(chunker.AddressToNames).To(Equal(map[string][]string{
-				"0x00000000000000000000000000000000000000a1": []string{"TransformerA"},
+				"0x00000000000000000000000000000000000000a1": {"TransformerA"},
-				"0x00000000000000000000000000000000000000a2": []string{"TransformerA", "TransformerC"},
+				"0x00000000000000000000000000000000000000a2": {"TransformerA", "TransformerC"},
-				"0x00000000000000000000000000000000000000b1": []string{"TransformerB"},
+				"0x00000000000000000000000000000000000000b1": {"TransformerB"},
 			}))
 
 			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{

libraries/shared/constants/data.go

@@ -17,6 +17,3 @@
 package constants
 
 var DataItemLength = 32
-
-// TODO Grab this from DB, since it can change through governance
-var TTL = int64(10800) // 60 * 60 * 3 == 10800 seconds == 3 hours

libraries/shared/factories/README.md

@@ -0,0 +1,388 @@
+# Watching Contract Events
+	
+One approach VulcanizeDB takes to caching and indexing smart contracts is to watch contract events emitted in receipt logs.
+
+With a light synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
+fetch and verify relevant event logs from a full Ethereum node, keeping track of which headers we have checked for which events 
+with our `checked_headers` table.
+
+## Assumptions
+
+This approach assumes you are running a vDB light sync which is itself run against a light Ethereum node,
+this approach also assumes there is a full node available. 
+
+Looking forward, we will be building fetchers that enable sourcing data from IPFS instead of an ETH node.
+
+## Shared Code
+
+VulcanizeDB has shared code built out for building and plugging in event transformers
+
+### [Event Watcher (light sync)](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/watcher/event_watcher.go)
+
+The event watcher is responsible for continuously fetching and delegating chunks of logs and their associated header to the appropriate transformers.
+
+Using the `compose` or `composeAndExecute` command, event watchers can be loaded with plugin event transformers and execute over them.
+
+### [Event Transformer](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/libraries/shared/transformer/event_transformer.go)
+
+The event transformer is responsible for converting event logs into more useful data objects and storing them in Postgres.
+The event transformer is composed of converter and repository interfaces and a config struct:
+```go
+type Transformer struct {
+	Config     transformer.TransformerConfig
+	Converter  Converter
+	Repository Repository
+}
+```
+
+The event transformer executes over provided event logs at a given header.
+
+In this process, the converter unpacks these logs into entities and then converts these entities 
+to their final db models. These models are then written to the Postgres db by the repository.
+
+```go
+func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+	transformerName := transformer.Config.TransformerName
+	config := transformer.Config
+
+	if len(logs) < 1 {
+		err := transformer.Repository.MarkHeaderChecked(header.Id)
+		if err != nil {
+			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
+			return err
+		}
+		return nil
+	}
+
+	entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
+	if err != nil {
+		log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
+		return err
+	}
+
+	models, err := transformer.Converter.ToModels(entities)
+	if err != nil {
+		log.Printf("Error converting entities to models in %v: %v", transformerName, err)
+		return err
+	}
+
+	err = transformer.Repository.Create(header.Id, models)
+	if err != nil {
+		log.Printf("Error persisting %v record: %v", transformerName, err)
+		return err
+	}
+
+	return nil
+}
+```
+
+
+## Custom Code
+
+In order to watch events at a smart contract, for those events the developer must create:
+
+1. Config - struct to hold configuration information (contract address, starting block, event name and signature).
+1. Entity - struct to unpack the event log into.
+1. Model - struct representing the final data model we want to write to Postgres.
+1. Converter - an interface which can unpack event logs into our entities and convert those entities to our models. 
+1. Repository - an interface to write our models to Postgres.
+1. TransformerInitializer - a public variable which exports our configured transformer to be loaded as part of a plugin.
+1. DB migrations - migrations to generate the Postgres schema, tables, views, function, etc that are needed to store and interface with the transformed data models.
+
+The example event we will use looks like: 
+```
+event ExampleEvent(bytes32 indexed arg1, address indexed arg2, bytes32 arg3, uint256 arg4, uint256 arg5);
+```
+
+### Config
+
+The config holds configuration variables for the event transformer, including a name for the transformer, the contract address
+it is working at, the contract's ABI, the topic (e.g. event signature; topic0) that it is filtering for, and starting
+and ending block numbers.
+
+```go
+type TransformerConfig struct {
+	TransformerName     string
+	ContractAddresses   []string
+	ContractAbi         string
+	Topic               string
+	StartingBlockNumber int64
+	EndingBlockNumber   int64 // Set -1 for indefinite transformer
+}
+```
+
+### Entity
+
+Entity field names for event arguments need to be exported and match the argument's name and type. LogIndex, 
+TransactionIndex, and the Raw log are retained in order to link the data to it's source for downstream validation.
+
+```go
+type ExampleEntity struct {
+	Arg1             common.Hash
+	Arg2             common.Address
+	Arg3             common.Hash
+	Arg4             *big.Int
+	Arg5             *big.Int
+	LogIndex         uint
+	TransactionIndex uint
+	Raw              types.Log
+}
+```
+
+### Model
+
+Model fields are not constrained by the event log structure.
+This allows us to rename our fields, decode or convert our log values into more useful types, and perform operations 
+with or on the values before persisting the data to Postgres.
+
+```go
+type ExampleModel struct {
+	EventHash         string
+	UserAddress       string
+	FractionSkimmed   string 
+	Surplus           string 
+	Deficit           string
+	FinalPosition     string
+	LogIndex          uint
+	TransactionIndex  uint
+	Raw               types.Log
+}
+```
+
+### Converter
+
+The converter needs to satisfy the interface. One for unpacking logs into the custom defined entities, and
+another for converting those entities to their final db models.
+
+```go
+type Converter interface {
+	ToEntities(contractAbi string, ethLog []types.Log) ([]interface{}, error)
+	ToModels([]interface{}) ([]interface{}, error)
+}
+```
+
+For the example event, this might look like:
+```go
+type ExampleConverter struct{}
+
+func (ExampleConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
+	var entities []interface{}
+	for _, ethLog := range ethLogs {
+		entity := &ExampleEntity{}
+		address := ethLog.Address
+		abi, err := geth.ParseAbi(contractAbi)
+		if err != nil {
+			return nil, err
+		}
+
+		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+
+		err = contract.UnpackLog(entity, "ExampleEvent", ethLog)
+		if err != nil {
+			return nil, err
+		}
+
+		entity.Raw = ethLog
+		entity.LogIndex = ethLog.Index
+		entity.TransactionIndex = ethLog.TxIndex
+
+		entities = append(entities, *entity)
+	}
+
+	return entities, nil
+}
+
+func (converter ExampleConverter) ToModels(entities []interface{}) ([]interface{}, error) {
+	var models []interface{}
+	for _, entity := range entities {
+        entity, ok := entity.(ExampleModel)
+        if !ok {
+            return nil, fmt.Errorf("entity of type %T, not %T", entity, ExampleModel{})
+        }
+        
+        fractionSkimmed, err := hexutil.DecodeBig(entity.Arg3.Hex())
+        if err != nil {
+        	reuturn nil, err
+        }
+        position := new(big.Int)
+        position.Sub(entity.Arg4, entity.Arg5)
+        finalPosition := new(big.Int)
+        if preTaxPosition.Sign() < 0 {
+            finalPosition = position
+        } else {
+            skim := new(big.Int)
+            skim.Div(position, fractionSkimmed)
+            finalPosition = position.Sub(position, skim)
+        }
+        
+        rawLog, err := json.Marshal(entity.Raw)
+        if err != nil {
+            return nil, err
+        }
+        
+        model := ExampleModel{
+            EventHash:        entity.Arg1.Hex(),
+            UserAddress:      entity.Arg2.Hex(),
+            FractionSkimmed:  fractionSkimmed.String(), 
+            Surplus:          entity.Arg4.String(),
+            Deficit:          entity.Arg5.String(),
+            FinalPosition:    finalPosition,
+            LogIndex:         entity.LogIndex,
+            TransactionIndex: entity.TransactionIndex,
+            Raw:              rawLog,
+        }
+        models = append(models, model)
+	}
+	return models, nil
+}
+```
+Notice that in this example we have a bytes32 argument in the event that needs to be decoded to an integer before it can be worked with
+to produce our hypothetical `FinalPosition` field. This is to highlight the fact that contracts can and sometimes do encode the
+data types we want to work with into raw bytes. Writing custom transformers with these converters allows us to account for this.
+
+### Repository
+
+The repository needs to satisfy the interface and use the `Create` method to write the model to Postgres.
+```go
+type Repository interface {
+	Create(headerID int64, models []interface{}) error
+	MarkHeaderChecked(headerID int64) error
+	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
+	RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
+	SetDB(db *postgres.DB)
+}
+```
+
+For the example event, this might look like:
+```go
+type ExampleRepository struct {
+	db *postgres.DB
+}
+
+func (repository *ExampleRepository) SetDB(db *postgres.DB) {
+	repository.db = db
+}
+
+func (repository ExampleRepository) Create(headerID int64, models []interface{}) error {
+	tx, dBaseErr := repository.db.Begin()
+	if dBaseErr != nil {
+		return dBaseErr
+	}
+	for _, model := range models {
+		model, ok := model.(ExampleModel)
+		if !ok {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				log.Error("failed to rollback ", rollbackErr)
+			}
+			return fmt.Errorf("model of type %T, not %T", model, ExampleModel{})
+		}
+		
+		_, execErr := tx.Exec(
+			`INSERT into example_schema.example_event (header_id, event_hash, user_address, fraction_skimmed, surplus, deficit, final_position, log_idx, tx_idx, raw_log)
+        			VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+					ON CONFLICT (header_id, tx_idx, log_idx) DO UPDATE SET event_hash = $2, user_address = $3, fraction_skimmed = $4, surplus = $5, deficit = $6, final_position = $7, raw_log = $10;`,
+			headerID, model.EventHash, model.UserAddress, model.FractonSkimmed, model.Surplus, model.Deficit, model.FinalPosition, model.LogIndex, model.TransactionIndex, model.Raw,
+		)
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				log.Error("failed to rollback ", rollbackErr)
+			}
+			return execErr
+		}
+	}
+
+	checkHeaderErr := repo.MarkHeaderCheckedInTransaction(headerID, tx, "example_event_checked")
+	if checkHeaderErr != nil {
+		rollbackErr := tx.Rollback()
+		if rollbackErr != nil {
+			log.Error("failed to rollback ", rollbackErr)
+		}
+		return checkHeaderErr
+	}
+
+	return tx.Commit()
+}
+
+func (repository ExampleRepository) MarkHeaderChecked(headerID int64) error {
+	return repo.MarkHeaderChecked(headerID, repository.db, "example_event_checked")
+}
+
+func (repository ExampleRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
+	return repo.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db,"example_event_checked")
+}
+
+func (repository ExampleRepository) RecheckHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
+	return repo.RecheckHeaders(startingBlockNumber, endingBlockNumber, repository.db, "example_event_checked")
+}
+```
+
+### TransformerInitializer
+
+A transformer initializer variable needs to be exported from somewhere within the transformer repository so that the transformer can be
+loaded as part of a plugin in the `compose` or `composeAndExecute` commands. It is important that this variable is named `TransformerInitializer` and
+it must be of `type TransformerInitializer func(db *postgres.DB) EventTransformer`.
+
+```go
+var TransformerInitializer transformer.TransformerInitializer = factories.Transformer{
+	Config:     exampleEventConfig,
+	Converter:  ExampleConverter{},
+	Repository: &ExampleRepository{},
+}.NewTransformer
+```
+
+### DB migrations
+
+We use `goose` as our migration management tool. Any Go data model that needs to be written to Postgres by the
+repository needs a db migration for the corresponding Postgres data model.
+
+Each contract or set of transformers being watched should define its own namespace with a db schema:
+```postgresql
+-- +goose Up
+CREATE SCHEMA example_schema;
+
+
+-- +goose Down
+DROP SCHEMA example_schema;
+
+```
+
+For the example event and its resulting model, the table we write to would look like:
+```postgresql
+-- +goose Up
+CREATE TABLE example_schema.example_event (
+  id               SERIAL PRIMARY KEY,
+  header_id        INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+  event_hash       CHARACTER VARYING(66) NOT NULL,
+  user_address     CHARACTER VARYING(66) NOT NULL,
+  fraction_skimmed NUMERIC NOT NULL,
+  surplus          NUMERIC NOT NULL,
+  deficit          NUMERIC NOT NULL,
+  final_position   NUMERIC NOT NULL,
+  tx_idx           INTEGER NOT NUll,
+  log_idx          INTEGER NOT NUll,
+  raw_log          JSONB,
+  UNIQUE (header_id, tx_idx, log_idx)
+);
+
+ALTER TABLE public.checked_headers
+  ADD COLUMN example_event_checked INTEGER NOT NULL DEFAULT 0;
+
+
+-- +goose Down
+DROP TABLE maker.bite;
+
+ALTER TABLE public.checked_headers
+  DROP COLUMN bite_checked;
+``` 
+
+Notice that we have also added a column to the `checked_headers` table for this event so that we can keep track
+of which headers we have already filtered through for this event.
+
+## Summary
+
+To create a transformer for a contract event we need to create entities for unpacking the raw log, models to represent
+the final data structure, a converter to mediate this unpacking and conversion between entities to models, a repository to write
+these models to Postgres, db migrations to accommodate these models in Postgres, and a TransformerInitializer to export the
+configured transformer and load it as a plugin to the `compose` or `composeAndExecute` commands as described in the [main readme](https://github.com/vulcanize/maker-vulcanizedb/blob/staging/README.md#composeandexecute-configuration).

libraries/shared/factories/storage/EXAMPLE.md

@@ -26,7 +26,7 @@ contract Contract {
   
   function add_address(address addr) public {
     bool exists = addresses[addr] > 0;
-    addresses[addr] = addresses[addr] + 1;
+    addresses[addr]++;
     if (!exists) {
       emit AddressAdded(addr, ++num_addresses);
     }
@@ -36,7 +36,7 @@ contract Contract {
 
 Disclaimer: this contract has not been audited and is not intended to be modeled or used in production. :)
 
-This contract persists two values in it's storage:
+This contract persists two values in its storage:
 
 1. `num_addresses`: the total number of unique addresses known to the contract.
 2. `addresses`: a mapping that records the number of times an address has been added to the contract.
@@ -131,9 +131,9 @@ Once we have recognized a storage diff, we can decode the storage value to the d
 Since the metadata tells us that the above values are `uint256`, we can decode a value like `0000000000000000000000000000000000000000000000000000000000000001` to `1`.
 
 The purpose of the contract-specific repository is to write that value to the database in a way that makes it useful for future queries.
-Typically, the involves writing the block hash, block number, decoded value, and any keys in the metadata to a table.
+Typically, this involves writing the block hash, block number, decoded value, and any keys in the metadata to a table.
 
-The current repository interface has a generalized `Create` function that can accept any arbitrary storage row along with it's metadata.
+The current repository interface has a generalized `Create` function that can accept any arbitrary storage row along with its metadata.
 This is deliberate, to facilitate shared use of the common storage transformer.
 An implication of this decision is that the `Create` function typically includes a `switch` statement that selects which table to write to, as well as what data to include, based on the name of the variable as defined in the metadata.
 
@@ -158,7 +158,7 @@ func (repository AddressStorageRepository) Create(blockNumber int, blockHash str
 
 ## Summary
 
-With our very simple address storing contract, we would be able to read it's storage diffs by implementing an event transformer, a mappings, and a repository.
+With our very simple address storing contract, we would be able to read its storage diffs by implementing an event transformer, a mappings, and a repository.
 
 The mappings would be able to lookup storage keys reflecting `num_addresses` or any slot in `addresses`, using addresses derived from watching the `AddressAdded` event for the latter.
 

libraries/shared/factories/storage/README.md

@@ -23,7 +23,7 @@ VulcanizeDB has shared code for continuously reading from the CSV file written b
 The storage watcher is responsible for continuously delegating CSV rows to the appropriate transformer as they are being written by the ethereum node.
 It maintains a mapping of contract addresses to transformers, and will ignore storage diff rows for contract addresses that do not have a corresponding transformer.
 
-The storage watcher is currently initialized from the `parseStorageDiffs` command, which also adds transformers that the watcher should know about in its mapping of addresses to transformers.
+Storage watchers can be loaded with plugin storage transformers and executed using the `composeAndExecute` command.
 
 ### Storage Transformer
 

libraries/shared/repository/repository.go

@@ -18,9 +18,9 @@ package repository
 
 import (
 	"bytes"
-	"database/sql"
 	"database/sql/driver"
 	"fmt"
+	"github.com/jmoiron/sqlx"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -35,7 +35,7 @@ func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn str
 	return err
 }
 
-func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, checkedHeadersColumn string) error {
+func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
 	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
 		VALUES ($1, $2)
 		ON CONFLICT (header_id) DO
@@ -155,14 +155,3 @@ func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.Transfor
 
 	return result.String()
 }
-
-func GetTicInTx(headerID int64, tx *sql.Tx) (int64, error) {
-	var blockTimestamp int64
-	err := tx.QueryRow(`SELECT block_timestamp FROM public.headers WHERE id = $1;`, headerID).Scan(&blockTimestamp)
-	if err != nil {
-		return 0, err
-	}
-
-	tic := blockTimestamp + constants.TTL
-	return tic, nil
-}

libraries/shared/repository/repository_test.go

@@ -19,6 +19,7 @@ package repository_test
 import (
 	"fmt"
 	"math/rand"
+	"strconv"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -34,7 +35,83 @@ import (
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
-var _ = Describe("Repository utilities", func() {
+var _ = Describe("Repository", func() {
+	Describe("MarkHeaderChecked", func() {
+		var (
+			checkedHeadersColumn string
+			db                   *postgres.DB
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+
+			checkedHeadersColumn = "test_column_checked"
+			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
+				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
+			Expect(migrateErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+		})
+
+		It("marks passed column as checked for passed header", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(headerErr).NotTo(HaveOccurred())
+
+			err := shared.MarkHeaderChecked(headerID, db, checkedHeadersColumn)
+
+			Expect(err).NotTo(HaveOccurred())
+			var checkedCount int
+			fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
+			Expect(fetchErr).NotTo(HaveOccurred())
+			Expect(checkedCount).To(Equal(1))
+		})
+	})
+
+	Describe("MarkHeaderCheckedInTransaction", func() {
+		var (
+			checkedHeadersColumn string
+			db                   *postgres.DB
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+
+			checkedHeadersColumn = "test_column_checked"
+			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
+				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
+			Expect(migrateErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+		})
+
+		It("marks passed column as checked for passed header within a passed transaction", func() {
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(headerErr).NotTo(HaveOccurred())
+			tx, txErr := db.Beginx()
+			Expect(txErr).NotTo(HaveOccurred())
+
+			err := shared.MarkHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
+
+			Expect(err).NotTo(HaveOccurred())
+			commitErr := tx.Commit()
+			Expect(commitErr).NotTo(HaveOccurred())
+			var checkedCount int
+			fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
+			Expect(fetchErr).NotTo(HaveOccurred())
+			Expect(checkedCount).To(Equal(1))
+		})
+	})
+
 	Describe("MissingHeaders", func() {
 		var (
 			db                       *postgres.DB
@@ -116,6 +193,84 @@ var _ = Describe("Repository utilities", func() {
 		})
 	})
 
+	Describe("RecheckHeaders", func() {
+		var (
+			checkedHeadersColumn                                      string
+			db                                                        *postgres.DB
+			headerOneID, headerTwoID, headerThreeID, headerFourID     int64
+			headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
+		)
+
+		BeforeEach(func() {
+			db = test_config.NewTestDB(test_config.NewTestNode())
+			test_config.CleanTestDB(db)
+
+			// create header checked column
+			checkedHeadersColumn = "test_column_checked"
+			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
+			Expect(migrateErr).NotTo(HaveOccurred())
+
+			// create headers
+			headerRepository := repositories.NewHeaderRepository(db)
+			headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
+			Expect(headerOneErr).NotTo(HaveOccurred())
+			headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
+			Expect(headerTwoErr).NotTo(HaveOccurred())
+			headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
+			Expect(headerThreeErr).NotTo(HaveOccurred())
+			headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
+			Expect(headerFourErr).NotTo(HaveOccurred())
+
+			// mark every header checked at least once, with one fully rechecked (headerThree)
+			maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
+			Expect(intConversionErr).NotTo(HaveOccurred())
+			_, markHeaderOneCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerOneID, maxCheckCount)
+			Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
+			_, markHeaderTwoCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerTwoID, maxCheckCount)
+			Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
+			_, markHeaderThreeCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerThreeID, maxCheckCount+1)
+			Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
+			_, markHeaderFourCheckedErr := db.Exec(
+				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
+				headerFourID, maxCheckCount)
+			Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+		})
+
+		Describe("when no ending block number (ending block number == -1)", func() {
+			It("returns all headers since starting block where checked count is less than cap", func() {
+				headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(3))
+				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+				Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+			})
+		})
+
+		Describe("when ending block number specified", func() {
+			It("returns headers between starting and ending block where checked count is less than cap", func() {
+				headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(headers)).To(Equal(2))
+				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
+				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
+			})
+		})
+	})
+
 	Describe("GetCheckedColumnNames", func() {
 		It("gets the column names from checked_headers", func() {
 			db := test_config.NewTestDB(test_config.NewTestNode())
@@ -162,33 +317,9 @@ var _ = Describe("Repository utilities", func() {
 
 func getExpectedColumnNames() []string {
 	return []string{
-		"price_feeds_checked",
+		"column_1_checked",
-		"flip_kick_checked",
+		"column_2_checked",
-		"frob_checked",
+		"column_3_checked",
-		"tend_checked",
+		"column_4_checked",
-		"bite_checked",
-		"dent_checked",
-		"pit_file_debt_ceiling_checked",
-		"pit_file_ilk_checked",
-		"vat_init_checked",
-		"drip_file_ilk_checked",
-		"drip_file_repo_checked",
-		"drip_file_vow_checked",
-		"deal_checked",
-		"drip_drip_checked",
-		"cat_file_chop_lump_checked",
-		"cat_file_flip_checked",
-		"cat_file_pit_vow_checked",
-		"flop_kick_checked",
-		"vat_move_checked",
-		"vat_fold_checked",
-		"vat_heal_checked",
-		"vat_toll_checked",
-		"vat_tune_checked",
-		"vat_grab_checked",
-		"vat_flux_checked",
-		"vat_slip_checked",
-		"vow_flog_checked",
-		"flap_kick_checked",
 	}
 }

libraries/shared/storage/mappings_test.go

@@ -0,0 +1,65 @@
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+)
+
+var _ = Describe("Mappings", func() {
+	Describe("GetMapping", func() {
+		It("returns the storage key for a mapping when passed the mapping's index on the contract and the desired value's key", func() {
+			// ex. solidity:
+			//    	mapping (bytes32 => uint) public amounts
+			// to access amounts, pass in the index of the mapping on the contract + the bytes32 key for the uint val being looked up
+			indexOfMappingOnContract := storage.IndexZero
+			keyForDesiredValueInMapping := "1234567890abcdef"
+
+			storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+
+			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
+	})
+
+	Describe("GetNestedMapping", func() {
+		It("returns the storage key for a nested mapping when passed the mapping's index on the contract and the desired value's keys", func() {
+			// ex. solidity:
+			//    	mapping (bytes32 => uint) public amounts
+			//    	mapping (address => mapping (uint => bytes32)) public addressNames
+			// to access addressNames, pass in the index of the mapping on the contract + the address and uint keys for the bytes32 val being looked up
+			indexOfMappingOnContract := storage.IndexOne
+			keyForOuterMapping := "1234567890abcdef"
+			keyForInnerMapping := "123"
+
+			storageKey := storage.GetNestedMapping(indexOfMappingOnContract, keyForOuterMapping, keyForInnerMapping)
+
+			expectedStorageKey := common.HexToHash("0x82113529f6cd61061d1a6f0de53f2bdd067a1addd3d2b46be50a99abfcdb1661")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
+	})
+
+	Describe("GetIncrementedKey", func() {
+		It("returns the storage key for later values sharing an index on the contract with other earlier values", func() {
+			// ex. solidity:
+			//    	mapping (bytes32 => uint) public amounts
+			//    	mapping (address => mapping (uint => bytes32)) public addressNames
+			//    	struct Data {
+			//        uint256 quantity;
+			//        uint256 quality;
+			//    	}
+			//    	mapping (bytes32 => Data) public itemData;
+			// to access quality from itemData, pass in the storage key for the zero-indexed value (quantity) + the number of increments required.
+			// (For "quality", we must increment the storage key for the corresponding "quantity" by 1).
+			indexOfMappingOnContract := storage.IndexTwo
+			keyForDesiredValueInMapping := "1234567890abcdef"
+			storageKeyForFirstPropertyOnStruct := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+
+			storageKey := storage.GetIncrementedKey(storageKeyForFirstPropertyOnStruct, 1)
+
+			expectedStorageKey := common.HexToHash("0x69b38749f0a8ed5d505c8474f7fb62c7828aad8a7627f1c67e07af1d2368cad4")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
+	})
+})

libraries/shared/storage/utils/decoder.go

@@ -27,6 +27,8 @@ func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, err
 	switch metadata.Type {
 	case Uint256:
 		return decodeUint256(row.StorageValue.Bytes()), nil
+	case Uint48:
+		return decodeUint48(row.StorageValue.Bytes()), nil
 	case Address:
 		return decodeAddress(row.StorageValue.Bytes()), nil
 	case Bytes32:
@@ -41,6 +43,11 @@ func decodeUint256(raw []byte) string {
 	return n.String()
 }
 
+func decodeUint48(raw []byte) string {
+	n := big.NewInt(0).SetBytes(raw)
+	return n.String()
+}
+
 func decodeAddress(raw []byte) string {
 	return common.BytesToAddress(raw).Hex()
 }

libraries/shared/storage/utils/decoder_test.go

@@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
 	})
 
+	It("decodes uint48", func() {
+		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
+		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		metadata := utils.StorageValueMetadata{Type: utils.Uint48}
+
+		result, err := utils.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
+	})
+
 	It("decodes address", func() {
 		fakeAddress := common.HexToAddress("0x12345")
 		row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()}

libraries/shared/storage/utils/value.go

@@ -20,6 +20,7 @@ type ValueType int
 
 const (
 	Uint256 ValueType = iota
+	Uint48
 	Bytes32
 	Address
 )

libraries/shared/storage/utils/value_test.go

@@ -0,0 +1,22 @@
+package utils_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
+
+var _ = Describe("Storage value metadata getter", func() {
+	It("returns a storage value metadata instance with corresponding fields assigned", func() {
+		metadataName := "fake_name"
+		metadataKeys := map[utils.Key]string{"key": "value"}
+		metadataType := utils.Uint256
+
+		expectedMetadata := utils.StorageValueMetadata{
+			Name: metadataName,
+			Keys: metadataKeys,
+			Type: metadataType,
+		}
+		Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
+	})
+})

libraries/shared/test_data/generic.go

@@ -49,7 +49,7 @@ var GenericTestConfig = transformer.TransformerConfig{
 }
 
 func randomString(length int) string {
-	var seededRand *rand.Rand = rand.New(
+	var seededRand = rand.New(
 		rand.NewSource(time.Now().UnixNano()))
 	charset := "abcdefghijklmnopqrstuvwxyz1234567890"
 	b := make([]byte, length)

libraries/shared/watcher/storage_watcher.go

@@ -50,8 +50,8 @@ func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
 
 func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
-		transformer := initializer(watcher.db)
+		storageTransformer := initializer(watcher.db)
-		watcher.Transformers[transformer.ContractAddress()] = transformer
+		watcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
 	}
 }
 
@@ -65,12 +65,12 @@ func (watcher StorageWatcher) Execute() error {
 		if parseErr != nil {
 			return parseErr
 		}
-		transformer, ok := watcher.Transformers[row.Contract]
+		storageTransformer, ok := watcher.Transformers[row.Contract]
 		if !ok {
 			logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
 			continue
 		}
-		executeErr := transformer.Execute(row)
+		executeErr := storageTransformer.Execute(row)
 		if executeErr != nil {
 			if isKeyNotFound(executeErr) {
 				queueErr := watcher.Queue.Add(row)

main.go

@@ -4,8 +4,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/cmd"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"os"
 )
 
@@ -19,9 +17,5 @@ func main() {
 		log.Info("Failed to log to file, using default stderr")
 	}
 
-	tracer.Start(tracer.WithServiceName(viper.GetString("datadog.name")))
-
 	cmd.Execute()
-
-	defer tracer.Stop()
 }

pkg/datastore/postgres/repositories/block_repository.go

@@ -17,7 +17,6 @@
 package repositories
 
 import (
-	"context"
 	"database/sql"
 	"errors"
 	log "github.com/sirupsen/logrus"
@@ -122,7 +121,7 @@ func (blockRepository BlockRepository) GetBlock(blockNumber int64) (core.Block,
 
 func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, error) {
 	var blockId int64
-	tx, _ := blockRepository.database.BeginTx(context.Background(), nil)
+	tx, _ := blockRepository.database.Beginx()
 	err := tx.QueryRow(
 		`INSERT INTO blocks
                 (eth_node_id, number, gaslimit, gasused, time, difficulty, hash, nonce, parenthash, size, uncle_hash, is_final, miner, extra_data, reward, uncles_reward, eth_node_fingerprint)
@@ -145,7 +144,7 @@ func (blockRepository BlockRepository) insertBlock(block core.Block) (int64, err
 	return blockId, nil
 }
 
-func (blockRepository BlockRepository) createTransactions(tx *sql.Tx, blockId int64, transactions []core.Transaction) error {
+func (blockRepository BlockRepository) createTransactions(tx *sqlx.Tx, blockId int64, transactions []core.Transaction) error {
 	for _, transaction := range transactions {
 		err := blockRepository.createTransaction(tx, blockId, transaction)
 		if err != nil {
@@ -165,7 +164,7 @@ func nullStringToZero(s string) string {
 	return s
 }
 
-func (blockRepository BlockRepository) createTransaction(tx *sql.Tx, blockId int64, transaction core.Transaction) error {
+func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId int64, transaction core.Transaction) error {
 	_, err := tx.Exec(
 		`INSERT INTO transactions
        (block_id, hash, nonce, tx_to, tx_from, gaslimit, gasprice, value, input_data)
@@ -198,7 +197,7 @@ func hasReceipt(transaction core.Transaction) bool {
 	return transaction.Receipt.TxHash != ""
 }
 
-func (blockRepository BlockRepository) createReceipt(tx *sql.Tx, blockId int64, receipt core.Receipt) (int, error) {
+func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) {
 	//Not currently persisting log bloom filters
 	var receiptId int
 	err := tx.QueryRow(
@@ -224,7 +223,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
 	return retrievedBlockHash, blockExists(retrievedBlockHash)
 }
 
-func (blockRepository BlockRepository) createLogs(tx *sql.Tx, logs []core.Log, receiptId int) error {
+func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error {
 	for _, tlog := range logs {
 		_, err := tx.Exec(
 			`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

pkg/datastore/postgres/repositories/logs_repository.go

@@ -17,7 +17,6 @@
 package repositories
 
 import (
-	"context"
 	"github.com/sirupsen/logrus"
 
 	"database/sql"
@@ -31,7 +30,7 @@ type LogRepository struct {
 }
 
 func (logRepository LogRepository) CreateLogs(lgs []core.Log, receiptId int64) error {
-	tx, _ := logRepository.DB.BeginTx(context.Background(), nil)
+	tx, _ := logRepository.DB.Beginx()
 	for _, tlog := range lgs {
 		_, err := tx.Exec(
 			`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)

pkg/datastore/postgres/repositories/receipt_repository.go

@@ -17,8 +17,8 @@
 package repositories
 
 import (
-	"context"
 	"database/sql"
+	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -31,7 +31,7 @@ type ReceiptRepository struct {
 }
 
 func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
-	tx, err := receiptRepository.DB.BeginTx(context.Background(), nil)
+	tx, err := receiptRepository.DB.Beginx()
 	if err != nil {
 		return err
 	}
@@ -53,7 +53,7 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64,
 	return nil
 }
 
-func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, error) {
+func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
 	var receiptId int64
 	err := tx.QueryRow(
 		`INSERT INTO receipts
@@ -68,7 +68,7 @@ func createReceipt(receipt core.Receipt, blockId int64, tx *sql.Tx) (int64, erro
 	return receiptId, err
 }
 
-func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
+func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
 	for _, log := range logs {
 		_, err := tx.Exec(
 			`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -84,7 +84,7 @@ func createLogs(logs []core.Log, receiptId int64, tx *sql.Tx) error {
 }
 
 func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
-	tx, _ := receiptRepository.DB.BeginTx(context.Background(), nil)
+	tx, _ := receiptRepository.DB.Beginx()
 	var receiptId int64
 	err := tx.QueryRow(
 		`INSERT INTO receipts

pkg/fakes/data.go

@@ -24,7 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
@@ -49,5 +48,3 @@ func GetFakeHeader(blockNumber int64) core.Header {
 		Timestamp:   strconv.FormatInt(fakeTimestamp, 10),
 	}
 }
-
-var FakeHeaderTic = fakeTimestamp + constants.TTL

pkg/fakes/mock_filter_repository.go

@@ -0,0 +1,14 @@
+package fakes
+
+import "github.com/vulcanize/vulcanizedb/pkg/filters"
+
+type MockFilterRepository struct {
+}
+
+func (*MockFilterRepository) CreateFilter(filter filters.LogFilter) error {
+	return nil
+}
+
+func (*MockFilterRepository) GetFilter(name string) (filters.LogFilter, error) {
+	panic("implement me")
+}

pkg/fakes/mock_full_block_retriever.go

@@ -0,0 +1,15 @@
+package fakes
+
+type MockFullBlockRetriever struct {
+	FirstBlock      int64
+	FirstBlockErr   error
+	MostRecentBlock int64
+}
+
+func (retriever *MockFullBlockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
+	return retriever.FirstBlock, retriever.FirstBlockErr
+}
+
+func (retriever *MockFullBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
+	return retriever.MostRecentBlock, nil
+}

pkg/fakes/mock_light_block_retriever.go

@@ -0,0 +1,14 @@
+package fakes
+
+type MockLightBlockRetriever struct {
+	FirstBlock    int64
+	FirstBlockErr error
+}
+
+func (retriever *MockLightBlockRetriever) RetrieveFirstBlock() (int64, error) {
+	return retriever.FirstBlock, retriever.FirstBlockErr
+}
+
+func (retriever *MockLightBlockRetriever) RetrieveMostRecentBlock() (int64, error) {
+	return 0, nil
+}

pkg/fakes/mock_light_header_repository.go

@@ -0,0 +1,42 @@
+package fakes
+
+import "github.com/vulcanize/vulcanizedb/pkg/core"
+
+type MockLightHeaderRepository struct {
+}
+
+func (*MockLightHeaderRepository) AddCheckColumn(id string) error {
+	return nil
+}
+
+func (*MockLightHeaderRepository) AddCheckColumns(ids []string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MarkHeaderChecked(headerID int64, eventID string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, eventID string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
+	panic("implement me")
+}
+
+func (*MockLightHeaderRepository) CheckCache(key string) (interface{}, bool) {
+	panic("implement me")
+}

pkg/fakes/mock_parser.go

@@ -0,0 +1,40 @@
+package fakes
+
+import (
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
+)
+
+type MockParser struct {
+	AbiToReturn string
+	EventName   string
+	Event       types.Event
+}
+
+func (*MockParser) Parse(contractAddr string) error {
+	return nil
+}
+
+func (*MockParser) ParseAbiStr(abiStr string) error {
+	panic("implement me")
+}
+
+func (parser *MockParser) Abi() string {
+	return parser.AbiToReturn
+}
+
+func (*MockParser) ParsedAbi() abi.ABI {
+	return abi.ABI{}
+}
+
+func (*MockParser) GetMethods(wanted []string) []types.Method {
+	panic("implement me")
+}
+
+func (*MockParser) GetSelectMethods(wanted []string) []types.Method {
+	return []types.Method{}
+}
+
+func (parser *MockParser) GetEvents(wanted []string) map[string]types.Event {
+	return map[string]types.Event{parser.EventName: parser.Event}
+}

pkg/fakes/mock_poller.go

@@ -0,0 +1,24 @@
+package fakes
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
+)
+
+type MockPoller struct {
+	ContractName string
+}
+
+func (*MockPoller) PollContract(con contract.Contract) error {
+	panic("implement me")
+}
+
+func (*MockPoller) PollContractAt(con contract.Contract, blockNumber int64) error {
+	panic("implement me")
+}
+
+func (poller *MockPoller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
+	if p, ok := result.(*string); ok {
+		*p = poller.ContractName
+	}
+	return nil
+}

pkg/omni/full/converter/converter.go

@@ -55,10 +55,10 @@ func (c *converter) Update(info *contract.Contract) {
 
 // Convert the given watched event log into a types.Log for the given event
 func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	values := make(map[string]interface{})
 	log := helpers.ConvertToLog(watchedEvent)
-	err := contract.UnpackLogIntoMap(values, event.Name, log)
+	err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 	if err != nil {
 		return nil, err
 	}

pkg/omni/full/transformer/transformer.go

@@ -34,7 +34,7 @@ import (
 )
 
 // Requires a fully synced vDB and a running eth node (or infura)
-type transformer struct {
+type Transformer struct {
 	// Database interfaces
 	datastore.FilterRepository       // Log filters repo; accepts filters generated by Contract.GenerateFilters()
 	datastore.WatchedEventRepository // Watched event log views, created by the log filters
@@ -76,8 +76,8 @@ type transformer struct {
 }
 
 // Transformer takes in config for blockchain, database, and network id
-func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transformer {
+func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *Transformer {
-	return &transformer{
+	return &Transformer{
 		Poller:                 poller.NewPoller(BC, DB, types.FullSync),
 		Parser:                 parser.NewParser(network),
 		BlockRetriever:         retriever.NewBlockRetriever(DB),
@@ -101,59 +101,59 @@ func NewTransformer(network string, BC core.BlockChain, DB *postgres.DB) *transf
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
 // Use this info to generate event filters
-func (t *transformer) Init() error {
+func (transformer *Transformer) Init() error {
-	for contractAddr, subset := range t.WatchedEvents {
+	for contractAddr, subset := range transformer.WatchedEvents {
 		// Get Abi
-		err := t.Parser.Parse(contractAddr)
+		err := transformer.Parser.Parse(contractAddr)
 		if err != nil {
 			return err
 		}
 
 		// Get first block and most recent block number in the header repo
-		firstBlock, err := t.BlockRetriever.RetrieveFirstBlock(contractAddr)
+		firstBlock, err := transformer.BlockRetriever.RetrieveFirstBlock(contractAddr)
 		if err != nil {
 			return err
 		}
-		lastBlock, err := t.BlockRetriever.RetrieveMostRecentBlock()
+		lastBlock, err := transformer.BlockRetriever.RetrieveMostRecentBlock()
 		if err != nil {
 			return err
 		}
 
 		// Set to specified range if it falls within the bounds
-		if firstBlock < t.ContractStart[contractAddr] {
+		if firstBlock < transformer.ContractStart[contractAddr] {
-			firstBlock = t.ContractStart[contractAddr]
+			firstBlock = transformer.ContractStart[contractAddr]
 		}
 
 		// Get contract name if it has one
 		var name = new(string)
-		t.FetchContractData(t.Abi(), contractAddr, "name", nil, &name, lastBlock)
+		transformer.Poller.FetchContractData(transformer.Abi(), contractAddr, "name", nil, name, lastBlock)
 
 		// Remove any potential accidental duplicate inputs in arg filter values
 		eventArgs := map[string]bool{}
-		for _, arg := range t.EventArgs[contractAddr] {
+		for _, arg := range transformer.EventArgs[contractAddr] {
 			eventArgs[arg] = true
 		}
 		methodArgs := map[string]bool{}
-		for _, arg := range t.MethodArgs[contractAddr] {
+		for _, arg := range transformer.MethodArgs[contractAddr] {
 			methodArgs[arg] = true
 		}
 
 		// Aggregate info into contract object
 		info := contract.Contract{
 			Name:           *name,
-			Network:        t.Network,
+			Network:        transformer.Network,
 			Address:        contractAddr,
-			Abi:            t.Parser.Abi(),
+			Abi:            transformer.Parser.Abi(),
-			ParsedAbi:      t.Parser.ParsedAbi(),
+			ParsedAbi:      transformer.Parser.ParsedAbi(),
 			StartingBlock:  firstBlock,
 			LastBlock:      lastBlock,
-			Events:         t.Parser.GetEvents(subset),
+			Events:         transformer.Parser.GetEvents(subset),
-			Methods:        t.Parser.GetSelectMethods(t.WantedMethods[contractAddr]),
+			Methods:        transformer.Parser.GetSelectMethods(transformer.WantedMethods[contractAddr]),
 			FilterArgs:     eventArgs,
 			MethodArgs:     methodArgs,
-			CreateAddrList: t.CreateAddrList[contractAddr],
+			CreateAddrList: transformer.CreateAddrList[contractAddr],
-			CreateHashList: t.CreateHashList[contractAddr],
+			CreateHashList: transformer.CreateHashList[contractAddr],
-			Piping:         t.Piping[contractAddr],
+			Piping:         transformer.Piping[contractAddr],
 		}.Init()
 
 		// Use info to create filters
@@ -164,14 +164,14 @@ func (t *transformer) Init() error {
 
 		// Iterate over filters and push them to the repo using filter repository interface
 		for _, filter := range info.Filters {
-			err = t.CreateFilter(filter)
+			err = transformer.FilterRepository.CreateFilter(filter)
 			if err != nil {
 				return err
 			}
 		}
 
 		// Store contract info for further processing
-		t.Contracts[contractAddr] = info
+		transformer.Contracts[contractAddr] = info
 	}
 
 	return nil
@@ -182,18 +182,18 @@ func (t *transformer) Init() error {
 // Uses converter to convert logs into custom log type
 // Persists converted logs into custuom postgres tables
 // Calls selected methods, using token holder address generated during event log conversion
-func (tr transformer) Execute() error {
+func (transformer Transformer) Execute() error {
-	if len(tr.Contracts) == 0 {
+	if len(transformer.Contracts) == 0 {
 		return errors.New("error: transformer has no initialized contracts to work with")
 	}
 	// Iterate through all internal contracts
-	for _, con := range tr.Contracts {
+	for _, con := range transformer.Contracts {
 		// Update converter with current contract
-		tr.Update(con)
+		transformer.Update(con)
 
 		// Iterate through contract filters and get watched event logs
 		for eventSig, filter := range con.Filters {
-			watchedEvents, err := tr.GetWatchedEvents(filter.Name)
+			watchedEvents, err := transformer.GetWatchedEvents(filter.Name)
 			if err != nil {
 				return err
 			}
@@ -201,7 +201,7 @@ func (tr transformer) Execute() error {
 			// Iterate over watched event logs
 			for _, we := range watchedEvents {
 				// Convert them to our custom log type
-				cstm, err := tr.Converter.Convert(*we, con.Events[eventSig])
+				cstm, err := transformer.Converter.Convert(*we, con.Events[eventSig])
 				if err != nil {
 					return err
 				}
@@ -211,7 +211,7 @@ func (tr transformer) Execute() error {
 
 				// If log is not empty, immediately persist in repo
 				// Run this in seperate goroutine?
-				err = tr.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
+				err = transformer.PersistLogs([]types.Log{*cstm}, con.Events[eventSig], con.Address, con.Name)
 				if err != nil {
 					return err
 				}
@@ -222,7 +222,7 @@ func (tr transformer) Execute() error {
 		// poller polls select contract methods
 		// and persists the results into custom pg tables
 		// Run this in seperate goroutine?
-		if err := tr.PollContract(*con); err != nil {
+		if err := transformer.PollContract(*con); err != nil {
 			return err
 		}
 	}
@@ -231,41 +231,41 @@ func (tr transformer) Execute() error {
 }
 
 // Used to set which contract addresses and which of their events to watch
-func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetEvents(contractAddr string, filterSet []string) {
-	tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
+	transformer.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to watch events for
-func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
-	tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
+	transformer.EventArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set which contract addresses and which of their methods to call
-func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetMethods(contractAddr string, filterSet []string) {
-	tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
+	transformer.WantedMethods[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to poll methods on
-func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
+func (transformer *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
-	tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
+	transformer.MethodArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set the block range to watch for a given address
-func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
+func (transformer *Transformer) SetStartingBlock(contractAddr string, start int64) {
-	tr.ContractStart[strings.ToLower(contractAddr)] = start
+	transformer.ContractStart[strings.ToLower(contractAddr)] = start
 }
 
 // Used to set whether or not to persist an account address list
-func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
+func (transformer *Transformer) SetCreateAddrList(contractAddr string, on bool) {
-	tr.CreateAddrList[strings.ToLower(contractAddr)] = on
+	transformer.CreateAddrList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to set whether or not to persist an hash list
-func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
+func (transformer *Transformer) SetCreateHashList(contractAddr string, on bool) {
-	tr.CreateHashList[strings.ToLower(contractAddr)] = on
+	transformer.CreateHashList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to turn method piping on for a contract
-func (tr *transformer) SetPiping(contractAddr string, on bool) {
+func (transformer *Transformer) SetPiping(contractAddr string, on bool) {
-	tr.Piping[strings.ToLower(contractAddr)] = on
+	transformer.Piping[strings.ToLower(contractAddr)] = on
 }

pkg/omni/full/transformer/transformer_test.go

@@ -17,353 +17,160 @@
 package transformer_test
 
 import (
-	"fmt"
 	"math/rand"
-	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/full/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/full/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/types"
 )
 
 var _ = Describe("Transformer", func() {
-	var db *postgres.DB
+	var fakeAddress = "0x1234567890abcdef"
-	var err error
-	var blockChain core.BlockChain
-	var blockRepository repositories.BlockRepository
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
 	rand.Seed(time.Now().UnixNano())
 
-	BeforeEach(func() {
-		db, blockChain = test_helpers.SetupDBandBC()
-		blockRepository = *repositories.NewBlockRepository(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
-
 	Describe("SetEvents", func() {
 		It("Sets which events to watch from the given contract address", func() {
 			watchedEvents := []string{"Transfer", "Mint"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEvents(constants.TusdContractAddress, watchedEvents)
+			t.SetEvents(fakeAddress, watchedEvents)
-			Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
+			Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
 		})
 	})
 
 	Describe("SetEventAddrs", func() {
 		It("Sets which account addresses to watch events for", func() {
 			eventAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
+			t.SetEventArgs(fakeAddress, eventAddrs)
-			Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
+			Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
 		})
 	})
 
 	Describe("SetMethods", func() {
 		It("Sets which methods to poll at the given contract address", func() {
 			watchedMethods := []string{"balanceOf", "totalSupply"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethods(constants.TusdContractAddress, watchedMethods)
+			t.SetMethods(fakeAddress, watchedMethods)
-			Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
+			Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
 		})
 	})
 
 	Describe("SetMethodAddrs", func() {
 		It("Sets which account addresses to poll methods against", func() {
 			methodAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
+			t.SetMethodArgs(fakeAddress, methodAddrs)
-			Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
+			Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
 		})
 	})
 
 	Describe("SetStartingBlock", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetStartingBlock(constants.TusdContractAddress, 11)
+			t.SetStartingBlock(fakeAddress, 11)
-			Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
+			Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
 		})
 	})
 
 	Describe("SetCreateAddrList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateAddrList(constants.TusdContractAddress, true)
+			t.SetCreateAddrList(fakeAddress, true)
-			Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("SetCreateHashList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateHashList(constants.TusdContractAddress, true)
+			t.SetCreateHashList(fakeAddress, true)
-			Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			blockRetriever := &fakes.MockFullBlockRetriever{}
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+			firstBlock := int64(1)
-			t := transformer.NewTransformer("", blockChain, db)
+			mostRecentBlock := int64(2)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			blockRetriever.FirstBlock = firstBlock
-			err = t.Init()
+			blockRetriever.MostRecentBlock = mostRecentBlock
+
+			parsr := &fakes.MockParser{}
+			fakeAbi := "fake_abi"
+			eventName := "Transfer"
+			event := types.Event{}
+			parsr.AbiToReturn = fakeAbi
+			parsr.EventName = eventName
+			parsr.Event = event
+
+			pollr := &fakes.MockPoller{}
+			fakeContractName := "fake_contract_name"
+			pollr.ContractName = fakeContractName
+
+			t := getTransformer(blockRetriever, parsr, pollr)
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			c, ok := t.Contracts[tusdAddr]
+			c, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(true))
 
-			Expect(c.StartingBlock).To(Equal(int64(6194633)))
+			Expect(c.StartingBlock).To(Equal(firstBlock))
-			Expect(c.LastBlock).To(Equal(int64(6194634)))
+			Expect(c.LastBlock).To(Equal(mostRecentBlock))
-			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(tusdAddr))
+			Expect(c.Address).To(Equal(fakeAddress))
 		})
 
 		It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			blockRetriever := &fakes.MockFullBlockRetriever{}
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			blockRetriever.FirstBlockErr = fakes.FakeError
-			err = t.Init()
+			t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
 		})
 
 		It("Does nothing if watched events are unset", func() {
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
+			t := getTransformer(&fakes.MockFullBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
+
-			t := transformer.NewTransformer("", blockChain, db)
+			err := t.Init()
-			err = t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			_, ok := t.Contracts[tusdAddr]
+			_, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(false))
 		})
 	})
-
-	Describe("Execute", func() {
-		BeforeEach(func() {
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock1)
-			blockRepository.CreateOrUpdateBlock(mocks.TransferBlock2)
 })
 
-		It("Transforms watched contract data into custom repositories", func() {
+func getTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
-			t := transformer.NewTransformer("", blockChain, db)
+	return transformer.Transformer{
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+		FilterRepository: &fakes.MockFilterRepository{},
-			t.SetMethods(constants.TusdContractAddress, nil)
+		Parser:           parsr,
-			err = t.Init()
+		BlockRetriever:   blockRetriever,
-			Expect(err).ToNot(HaveOccurred())
+		Poller:           pollr,
-
+		Contracts:        map[string]*contract.Contract{},
-			err = t.Execute()
+		WatchedEvents:    map[string][]string{},
-			Expect(err).ToNot(HaveOccurred())
+		WantedMethods:    map[string][]string{},
-
+		ContractStart:    map[string]int64{},
-			log := test_helpers.TransferLog{}
+		EventArgs:        map[string][]string{},
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.transfer_event WHERE block = 6194634", tusdAddr)).StructScan(&log)
+		MethodArgs:       map[string][]string{},
-
+		CreateAddrList:   map[string]bool{},
-			// We don't know vulcID, so compare individual fields instead of complete structures
+		CreateHashList:   map[string]bool{},
-			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654eee"))
+	}
-			Expect(log.Block).To(Equal(int64(6194634)))
+}
-			Expect(log.From).To(Equal("0x000000000000000000000000000000000000Af21"))
-			Expect(log.To).To(Equal("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"))
-			Expect(log.Value).To(Equal("1097077688018008265106216665536940668749033598146"))
-		})
-
-		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			b, ok := c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843bCE061BA391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.BalanceOf{}
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x000000000000000000000000000000000000Af21' AND block = '6194634'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("0"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843bCE061BA391' AND block = '6194634'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("0"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6194634'", tusdAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("Fails if initialization has not been done", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-
-			err = t.Execute()
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("Execute- against ENS registry contract", func() {
-		BeforeEach(func() {
-			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock1)
-			blockRepository.CreateOrUpdateBlock(mocks.NewOwnerBlock2)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.NewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
-
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.Tx).To(Equal("0x135391a0962a63944e5908e6fedfff90fb4be3e3290a21017861099bad654bbb"))
-			Expect(log.Block).To(Equal(int64(6194635)))
-			Expect(log.Node).To(Equal("0x0000000000000000000000000000000000000000000000000000c02aaa39b223"))
-			Expect(log.Label).To(Equal("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391"))
-			Expect(log.Owner).To(Equal("0x000000000000000000000000000000000000Af21"))
-		})
-
-		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			c, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedHashes)).To(Equal(3))
-
-			b, ok := c.EmittedHashes[common.HexToHash("0x0000000000000000000000000000000000000000000000000000c02aaa39b223")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedHashes[common.HexToHash("0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x000000000000000000000000000000000000Af21")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("It does not perist events if they do not pass the emitted arg filter", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
-
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("If a method arg filter is applied, only those arguments are used in polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x0000000000000000000000000000000000000000000000000000c02aaa39b223"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x0000000000000000000000000000000000000000000000000000c02aaa39b223' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.owner_method WHERE node_ = '0x9dd48110dcc444fdc242510c09bbbbe21a5975cac061d82f7b843bce061ba391' AND block = '6194636'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-})

pkg/omni/light/converter/converter.go

@@ -54,7 +54,7 @@ func (c *converter) Update(info *contract.Contract) {
 
 // Convert the given watched event log into a types.Log for the given event
 func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	returnLogs := make([]types.Log, 0, len(logs))
 	for _, log := range logs {
 		values := make(map[string]interface{})
@@ -63,7 +63,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 			values[field.Name] = i
 		}
 
-		err := contract.UnpackLogIntoMap(values, event.Name, log)
+		err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 		if err != nil {
 			return nil, err
 		}
@@ -133,7 +133,7 @@ func (c *converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 
 // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
 func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	eventsToLogs := make(map[string][]types.Log)
 	for _, event := range events {
 		eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@@ -142,7 +142,7 @@ func (c *converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
 			// If the log is of this event type, process it as such
 			if event.Sig() == log.Topics[0] {
 				values := make(map[string]interface{})
-				err := contract.UnpackLogIntoMap(values, event.Name, log)
+				err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 				if err != nil {
 					return nil, err
 				}

pkg/omni/light/repository/header_repository.go

@@ -17,8 +17,8 @@
 package repository
 
 import (
-	"database/sql"
 	"fmt"
+	"github.com/jmoiron/sqlx"
 
 	"github.com/hashicorp/golang-lru"
 
@@ -125,7 +125,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
 }
 
 func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
-	tx, err := r.db.Begin()
+	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
 	}
@@ -250,7 +250,7 @@ func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
 	return r.columns.Get(key)
 }
 
-func MarkHeaderCheckedInTransaction(headerID int64, tx *sql.Tx, eventID string) error {
+func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
 	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
 		VALUES ($1, $2) 
 		ON CONFLICT (header_id) DO

pkg/omni/light/retriever/retriever_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 func TestRetriever(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light BLock Number Retriever Suite Test")
+	RunSpecs(t, "Light Block Number Retriever Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/omni/light/transformer/transformer.go

@@ -37,7 +37,7 @@ import (
 )
 
 // Requires a light synced vDB (headers) and a running eth node (or infura)
-type transformer struct {
+type Transformer struct {
 	// Database interfaces
 	srep.EventRepository        // Holds transformed watched event log data
 	repository.HeaderRepository // Interface for interaction with header repositories
@@ -93,9 +93,9 @@ type transformer struct {
 // 4. Execute
 
 // Transformer takes in config for blockchain, database, and network id
-func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transformer {
+func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *Transformer {
 
-	return &transformer{
+	return &Transformer{
 		Poller:           poller.NewPoller(bc, db, types.LightSync),
 		Fetcher:          fetcher.NewFetcher(bc),
 		Parser:           parser.NewParser(network),
@@ -120,7 +120,7 @@ func NewTransformer(network string, bc core.BlockChain, db *postgres.DB) *transf
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
 // Use this info to generate event filters
-func (tr *transformer) Init() error {
+func (tr *Transformer) Init() error {
 	// Initialize internally configured transformer settings
 	tr.contractAddresses = make([]string, 0)       // Holds all contract addresses, for batch fetching of logs
 	tr.sortedEventIds = make(map[string][]string)  // Map to sort event column ids by contract, for post fetch processing and persisting of logs
@@ -154,7 +154,7 @@ func (tr *transformer) Init() error {
 
 		// Get contract name if it has one
 		var name = new(string)
-		tr.FetchContractData(tr.Abi(), contractAddr, "name", nil, &name, lastBlock)
+		tr.Poller.FetchContractData(tr.Abi(), contractAddr, "name", nil, name, lastBlock)
 
 		// Remove any potential accidental duplicate inputs in arg filter values
 		eventArgs := map[string]bool{}
@@ -221,7 +221,7 @@ func (tr *transformer) Init() error {
 	return nil
 }
 
-func (tr *transformer) Execute() error {
+func (tr *Transformer) Execute() error {
 	if len(tr.Contracts) == 0 {
 		return errors.New("error: transformer has no initialized contracts")
 	}
@@ -311,7 +311,7 @@ func (tr *transformer) Execute() error {
 }
 
 // Used to poll contract methods at a given header
-func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
+func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[string][]string) error {
 	for _, con := range tr.Contracts {
 		// Skip method polling processes if no methods are specified
 		// Also don't try to poll methods below this contract's specified starting block
@@ -336,41 +336,41 @@ func (tr *transformer) methodPolling(header core.Header, sortedMethodIds map[str
 }
 
 // Used to set which contract addresses and which of their events to watch
-func (tr *transformer) SetEvents(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetEvents(contractAddr string, filterSet []string) {
 	tr.WatchedEvents[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to watch events for
-func (tr *transformer) SetEventArgs(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetEventArgs(contractAddr string, filterSet []string) {
 	tr.EventArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set which contract addresses and which of their methods to call
-func (tr *transformer) SetMethods(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetMethods(contractAddr string, filterSet []string) {
 	tr.WantedMethods[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set subset of account addresses to poll methods on
-func (tr *transformer) SetMethodArgs(contractAddr string, filterSet []string) {
+func (tr *Transformer) SetMethodArgs(contractAddr string, filterSet []string) {
 	tr.MethodArgs[strings.ToLower(contractAddr)] = filterSet
 }
 
 // Used to set the block range to watch for a given address
-func (tr *transformer) SetStartingBlock(contractAddr string, start int64) {
+func (tr *Transformer) SetStartingBlock(contractAddr string, start int64) {
 	tr.ContractStart[strings.ToLower(contractAddr)] = start
 }
 
 // Used to set whether or not to persist an account address list
-func (tr *transformer) SetCreateAddrList(contractAddr string, on bool) {
+func (tr *Transformer) SetCreateAddrList(contractAddr string, on bool) {
 	tr.CreateAddrList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to set whether or not to persist an hash list
-func (tr *transformer) SetCreateHashList(contractAddr string, on bool) {
+func (tr *Transformer) SetCreateHashList(contractAddr string, on bool) {
 	tr.CreateHashList[strings.ToLower(contractAddr)] = on
 }
 
 // Used to turn method piping on for a contract
-func (tr *transformer) SetPiping(contractAddr string, on bool) {
+func (tr *Transformer) SetPiping(contractAddr string, on bool) {
 	tr.Piping[strings.ToLower(contractAddr)] = on
 }

pkg/omni/light/transformer/transformer_test.go

@@ -17,491 +17,150 @@
 package transformer_test
 
 import (
-	"fmt"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/light/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/light/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/constants"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/parser"
-	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/poller"
 )
 
 var _ = Describe("Transformer", func() {
-	var db *postgres.DB
+	var fakeAddress = "0x1234567890abcdef"
-	var err error
-	var blockChain core.BlockChain
-	var headerRepository repositories.HeaderRepository
-	var headerID, headerID2 int64
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
-
-	BeforeEach(func() {
-		db, blockChain = test_helpers.SetupDBandBC()
-		headerRepository = repositories.NewHeaderRepository(db)
-	})
-
-	AfterEach(func() {
-		test_helpers.TearDown(db)
-	})
 
 	Describe("SetEvents", func() {
 		It("Sets which events to watch from the given contract address", func() {
 			watchedEvents := []string{"Transfer", "Mint"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEvents(constants.TusdContractAddress, watchedEvents)
+			t.SetEvents(fakeAddress, watchedEvents)
-			Expect(t.WatchedEvents[tusdAddr]).To(Equal(watchedEvents))
+			Expect(t.WatchedEvents[fakeAddress]).To(Equal(watchedEvents))
 		})
 	})
 
 	Describe("SetEventAddrs", func() {
 		It("Sets which account addresses to watch events for", func() {
 			eventAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetEventArgs(constants.TusdContractAddress, eventAddrs)
+			t.SetEventArgs(fakeAddress, eventAddrs)
-			Expect(t.EventArgs[tusdAddr]).To(Equal(eventAddrs))
+			Expect(t.EventArgs[fakeAddress]).To(Equal(eventAddrs))
 		})
 	})
 
 	Describe("SetMethods", func() {
 		It("Sets which methods to poll at the given contract address", func() {
 			watchedMethods := []string{"balanceOf", "totalSupply"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethods(constants.TusdContractAddress, watchedMethods)
+			t.SetMethods(fakeAddress, watchedMethods)
-			Expect(t.WantedMethods[tusdAddr]).To(Equal(watchedMethods))
+			Expect(t.WantedMethods[fakeAddress]).To(Equal(watchedMethods))
 		})
 	})
 
 	Describe("SetMethodAddrs", func() {
 		It("Sets which account addresses to poll methods against", func() {
 			methodAddrs := []string{"test1", "test2"}
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetMethodArgs(constants.TusdContractAddress, methodAddrs)
+			t.SetMethodArgs(fakeAddress, methodAddrs)
-			Expect(t.MethodArgs[tusdAddr]).To(Equal(methodAddrs))
+			Expect(t.MethodArgs[fakeAddress]).To(Equal(methodAddrs))
 		})
 	})
 
 	Describe("SetStartingBlock", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetStartingBlock(constants.TusdContractAddress, 11)
+			t.SetStartingBlock(fakeAddress, 11)
-			Expect(t.ContractStart[tusdAddr]).To(Equal(int64(11)))
+			Expect(t.ContractStart[fakeAddress]).To(Equal(int64(11)))
 		})
 	})
 
 	Describe("SetCreateAddrList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateAddrList(constants.TusdContractAddress, true)
+			t.SetCreateAddrList(fakeAddress, true)
-			Expect(t.CreateAddrList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateAddrList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("SetCreateHashList", func() {
 		It("Sets the block range that the contract should be watched within", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			t.SetCreateHashList(constants.TusdContractAddress, true)
+			t.SetCreateHashList(fakeAddress, true)
-			Expect(t.CreateHashList[tusdAddr]).To(Equal(true))
+			Expect(t.CreateHashList[fakeAddress]).To(Equal(true))
 		})
 	})
 
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			blockRetriever := &fakes.MockLightBlockRetriever{}
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			firstBlock := int64(1)
-			t := transformer.NewTransformer("", blockChain, db)
+			blockRetriever.FirstBlock = firstBlock
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+
-			err = t.Init()
+			parsr := &fakes.MockParser{}
+			fakeAbi := "fake_abi"
+			parsr.AbiToReturn = fakeAbi
+
+			pollr := &fakes.MockPoller{}
+			fakeContractName := "fake_contract_name"
+			pollr.ContractName = fakeContractName
+
+			t := getFakeTransformer(blockRetriever, parsr, pollr)
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			c, ok := t.Contracts[tusdAddr]
+			c, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(true))
 
-			Expect(c.StartingBlock).To(Equal(int64(6194632)))
+			Expect(c.StartingBlock).To(Equal(firstBlock))
 			Expect(c.LastBlock).To(Equal(int64(-1)))
-			Expect(c.Abi).To(Equal(constants.TusdAbiString))
+			Expect(c.Abi).To(Equal(fakeAbi))
-			Expect(c.Name).To(Equal("TrueUSD"))
+			Expect(c.Name).To(Equal(fakeContractName))
-			Expect(c.Address).To(Equal(tusdAddr))
+			Expect(c.Address).To(Equal(fakeAddress))
 		})
 
 		It("Fails to initialize if first and most recent block numbers cannot be fetched from vDB headers table", func() {
-			t := transformer.NewTransformer("", blockChain, db)
+			blockRetriever := &fakes.MockLightBlockRetriever{}
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+			blockRetriever.FirstBlockErr = fakes.FakeError
-			err = t.Init()
+			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
+			t.SetEvents(fakeAddress, []string{"Transfer"})
+
+			err := t.Init()
+
 			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
 		})
 
 		It("Does nothing if watched events are unset", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			t := getFakeTransformer(&fakes.MockLightBlockRetriever{}, &fakes.MockParser{}, &fakes.MockPoller{})
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+
-			t := transformer.NewTransformer("", blockChain, db)
+			err := t.Init()
-			err = t.Init()
+
 			Expect(err).ToNot(HaveOccurred())
 
-			_, ok := t.Contracts[tusdAddr]
+			_, ok := t.Contracts[fakeAddress]
 			Expect(ok).To(Equal(false))
 		})
 	})
 
-	Describe("Execute- against TrueUSD contract", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6791668)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6791669)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6791670)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
 })
 
-		It("Transforms watched contract data into custom repositories", func() {
+func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Parser, pollr poller.Poller) transformer.Transformer {
-			t := transformer.NewTransformer("", blockChain, db)
+	return transformer.Transformer{
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
+		Parser:           parsr,
-			t.SetMethods(constants.TusdContractAddress, nil)
+		BlockRetriever:   blockRetriever,
-			err = t.Init()
+		Poller:           pollr,
-			Expect(err).ToNot(HaveOccurred())
+		HeaderRepository: &fakes.MockLightHeaderRepository{},
-			err = t.Execute()
+		Contracts:        map[string]*contract.Contract{},
-			Expect(err).ToNot(HaveOccurred())
+		WatchedEvents:    map[string][]string{},
-
+		WantedMethods:    map[string][]string{},
-			log := test_helpers.LightTransferLog{}
+		ContractStart:    map[string]int64{},
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
+		EventArgs:        map[string][]string{},
-			Expect(err).ToNot(HaveOccurred())
+		MethodArgs:       map[string][]string{},
-			// We don't know vulcID, so compare individual fields instead of complete structures
+		CreateAddrList:   map[string]bool{},
-			Expect(log.HeaderID).To(Equal(headerID))
+		CreateHashList:   map[string]bool{},
-			Expect(log.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
+	}
-			Expect(log.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
+}
-			Expect(log.Value).To(Equal("9998940000000000000000"))
-		})
-
-		It("Keeps track of contract-related addresses while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			c, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedAddrs)).To(Equal(4))
-			Expect(len(c.EmittedHashes)).To(Equal(0))
-
-			b, ok := c.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09BbBBE21a5975cAc061D82f7b843b1234567890")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x")]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[""]
-			Expect(ok).To(Equal(false))
-
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given methods using generated token holder address", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Balance).To(Equal("55849938025000000000000"))
-			Expect(res.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("Fails if initialization has not been done", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-			err = t.Execute()
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("Execute- against ENS registry contract", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6885695)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6885696)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6885697)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(log.HeaderID).To(Equal(headerID))
-			Expect(log.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
-			Expect(log.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
-			Expect(log.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-		})
-
-		It("Keeps track of contract-related hashes while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			c, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(c.EmittedHashes)).To(Equal(2))
-			Expect(len(c.EmittedAddrs)).To(Equal(0))
-
-			b, ok := c.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = c.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			// Doesn't keep track of address since it wouldn't be used in calling the 'owner' method
-			_, ok = c.EmittedAddrs[common.HexToAddress("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef")]
-			Expect(ok).To(Equal(false))
-		})
-
-		It("Polls given method using list of collected hashes", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("It does not persist events if they do not pass the emitted arg filter", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			t.SetEventArgs(constants.EnsContractAddress, []string{"fake_filter_value"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("If a method arg filter is applied, only those arguments are used in polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetMethodArgs(constants.EnsContractAddress, []string{"0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(res.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("Execute- against both ENS and TrueUSD", func() {
-		BeforeEach(func() {
-			header1, err := blockChain.GetHeaderByNumber(6791668)
-			Expect(err).ToNot(HaveOccurred())
-			header2, err := blockChain.GetHeaderByNumber(6791669)
-			Expect(err).ToNot(HaveOccurred())
-			header3, err := blockChain.GetHeaderByNumber(6791670)
-			Expect(err).ToNot(HaveOccurred())
-			header4, err := blockChain.GetHeaderByNumber(6885695)
-			Expect(err).ToNot(HaveOccurred())
-			header5, err := blockChain.GetHeaderByNumber(6885696)
-			Expect(err).ToNot(HaveOccurred())
-			header6, err := blockChain.GetHeaderByNumber(6885697)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header1)
-			headerID, err = headerRepository.CreateOrUpdateHeader(header2)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header3)
-			headerRepository.CreateOrUpdateHeader(header4)
-			headerID2, err = headerRepository.CreateOrUpdateHeader(header5)
-			Expect(err).ToNot(HaveOccurred())
-			headerRepository.CreateOrUpdateHeader(header6)
-		})
-
-		It("Transforms watched contract data into custom repositories", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, nil)
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, nil)
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			newOwnerLog := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(newOwnerLog.HeaderID).To(Equal(headerID2))
-			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
-			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
-			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-
-			transferLog := test_helpers.LightTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
-			Expect(err).ToNot(HaveOccurred())
-			// We don't know vulcID, so compare individual fields instead of complete structures
-			Expect(transferLog.HeaderID).To(Equal(headerID))
-			Expect(transferLog.From).To(Equal("0x1062a747393198f70F71ec65A582423Dba7E5Ab3"))
-			Expect(transferLog.To).To(Equal("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0"))
-			Expect(transferLog.Value).To(Equal("9998940000000000000000"))
-		})
-
-		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			ens, ok := t.Contracts[ensAddr]
-			Expect(ok).To(Equal(true))
-			tusd, ok := t.Contracts[tusdAddr]
-			Expect(ok).To(Equal(true))
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(ens.EmittedHashes)).To(Equal(2))
-			Expect(len(ens.EmittedAddrs)).To(Equal(0))
-			Expect(len(tusd.EmittedAddrs)).To(Equal(4))
-			Expect(len(tusd.EmittedHashes)).To(Equal(0))
-
-			b, ok := ens.EmittedHashes[common.HexToHash("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = ens.EmittedHashes[common.HexToHash("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x1062a747393198f70F71ec65A582423Dba7E5Ab3")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x2930096dB16b4A44Ecd4084EA4bd26F7EeF1AEf0")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0x571A326f5B15E16917dC17761c340c1ec5d06f6d")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-
-			b, ok = tusd.EmittedAddrs[common.HexToAddress("0xFBb1b73C4f0BDa4f67dcA266ce6Ef42f520fBB98")]
-			Expect(ok).To(Equal(true))
-			Expect(b).To(Equal(true))
-		})
-
-		It("Polls given methods for each contract, using list of collected values", func() {
-			t := transformer.NewTransformer("", blockChain, db)
-			t.SetEvents(constants.EnsContractAddress, []string{"NewOwner"})
-			t.SetMethods(constants.EnsContractAddress, []string{"owner"})
-			t.SetEvents(constants.TusdContractAddress, []string{"Transfer"})
-			t.SetMethods(constants.TusdContractAddress, []string{"balanceOf"})
-			err = t.Init()
-			Expect(err).ToNot(HaveOccurred())
-			err = t.Execute()
-			Expect(err).ToNot(HaveOccurred())
-
-			owner := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
-			Expect(owner.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
-			Expect(owner.TokenName).To(Equal(""))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
-			Expect(err).To(HaveOccurred())
-
-			bal := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&bal)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal.Balance).To(Equal("55849938025000000000000"))
-			Expect(bal.TokenName).To(Equal("TrueUSD"))
-
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&bal)
-			Expect(err).To(HaveOccurred())
-		})
-	})
-})

pkg/omni/shared/constants/constants.go

@@ -72,6 +72,7 @@ var TusdContractAddress = "0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E"
 var EnsContractAddress = "0x314159265dD8dbb310642f98f50C066173C1259b"
 var PublicResolverAddress = "0x1da022710dF5002339274AaDEe8D58218e9D6AB5"
 
+// TODO: Consider whether these should be moved to plugins
 // Contract Owner
 var DaiContractOwner = "0x0000000000000000000000000000000000000000"
 var TusdContractOwner = "0x9978d2d229a69b3aef93420d132ab22b44e3578f"

pkg/omni/shared/helpers/test_helpers/database.go

@@ -106,6 +106,7 @@ type Owner struct {
 	Address   string `db:"returned"`
 }
 
+// TODO: consider whether this should be moved to libraries/shared
 func SetupBC() core.BlockChain {
 	infuraIPC := "https://mainnet.infura.io/v3/b09888c1113640cc9ab42750ce750c05"
 	rawRpcClient, err := rpc.Dial(infuraIPC)
@@ -113,9 +114,9 @@ func SetupBC() core.BlockChain {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	blockChainNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
 
 	return blockChain
 }
@@ -127,9 +128,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	blockChainNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
 
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",
@@ -188,6 +189,7 @@ func SetupTusdContract(wantedEvents, wantedMethods []string) *contract.Contract
 	}.Init()
 }
 
+// TODO: consider whether this can be moved to plugin or libraries/shared
 func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (*postgres.DB, *contract.Contract) {
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",
@@ -236,7 +238,7 @@ func SetupENSContract(wantedEvents, wantedMethods []string) *contract.Contract {
 }
 
 func TearDown(db *postgres.DB) {
-	tx, err := db.Begin()
+	tx, err := db.Beginx()
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DELETE FROM blocks`)

pkg/omni/shared/repository/event_repository.go

@@ -97,7 +97,7 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
 
 // Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
 func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Begin()
+	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
 	}
@@ -151,7 +151,7 @@ func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types
 
 // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
 func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Begin()
+	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
 	}

pkg/omni/shared/repository/method_repository.go

@@ -77,7 +77,7 @@ func (r *methodRepository) PersistResults(results []types.Result, methodInfo typ
 
 // Creates a custom postgres command to persist logs for the given event
 func (r *methodRepository) persistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
-	tx, err := r.DB.Begin()
+	tx, err := r.DB.Beginx()
 	if err != nil {
 		return err
 	}

pkg/plugin/test_helpers/database.go

@@ -31,6 +31,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 )
 
+// TODO: consider whether this should be moved to libraries/shared
 func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	infuraIPC := "http://kovan0.vulcanize.io:8545"
 	rawRpcClient, err := rpc.Dial(infuraIPC)
@@ -38,9 +39,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	blockChainNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, blockChainNode, transactionConverter)
 
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",

scripts/start_blockchain

@@ -1,2 +0,0 @@
-#!/bin/bash
-geth --fast --cache=1024

scripts/start_test_chain.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-MNEMONIC_PHRASE="whisper ordinary mystery awesome wood fox auction february blind volcano spare soft"
-PORT=7545
-DATABASE_PATH=test_data/test_chain/
-echo Starting ganache chain on port $PORT...
-
-ganache-cli --port $PORT \
-            --db $DATABASE_PATH \
-            2>&1 > ganache-output.log &

scripts/stop_test_chain.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-echo "Stopping ganache chain on port 7545"
-ps -ef | grep ganache | grep -v grep | awk '{print $2}' | xargs kill

test_config/test_config.go

@@ -95,10 +95,13 @@ func NewTestDB(node core.Node) *postgres.DB {
 
 func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM blocks")
-	db.MustExec("DELETE FROM headers")
 	db.MustExec("DELETE FROM checked_headers")
+	// can't delete from eth_nodes since this function is called after the required eth_node is persisted
+	db.MustExec("DELETE FROM goose_db_version")
+	db.MustExec("DELETE FROM headers")
 	db.MustExec("DELETE FROM log_filters")
 	db.MustExec("DELETE FROM logs")
+	db.MustExec("DELETE FROM queued_storage")
 	db.MustExec("DELETE FROM receipts")
 	db.MustExec("DELETE FROM transactions")
 	db.MustExec("DELETE FROM watched_contracts")