Merge pull request #150 from vulcanize/unique-headers-constraint-v2

Add constraint to prevent duplicate headers

cmd/coldImport.go

@@ -23,10 +23,10 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/crypto"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/ethereum"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/cold_import"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/converters/cold_db"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/cold_import"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/cold_db"
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 

cmd/composeAndExecute.go

@@ -17,6 +17,10 @@
 package cmd
 
 import (
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"os"
 	"plugin"
 	syn "sync"
@@ -25,9 +29,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
 	"github.com/vulcanize/vulcanizedb/utils"
@@ -179,12 +181,26 @@ func composeAndExecute() {
 	}
 
 	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
-		sw.AddTransformers(ethStorageInitializers)
-		wg.Add(1)
-		go watchEthStorage(&sw, &wg)
+		switch storageDiffsSource {
+		case "geth":
+			log.Debug("fetching storage diffs from geth pub sub")
+			rpcClient, _ := getClients()
+			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
+			payloadChan := make(chan statediff.Payload)
+			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		default:
+			log.Debug("fetching storage diffs from csv")
+			tailer := fs.FileTailer{Path: storageDiffsPath}
+			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		}
 	}
 
 	if len(ethContractInitializers) > 0 {

cmd/execute.go

@@ -18,6 +18,10 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"plugin"
 	syn "sync"
 	"time"
@@ -26,11 +30,9 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -123,12 +125,26 @@ func execute() {
 	}
 
 	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
-		sw.AddTransformers(ethStorageInitializers)
-		wg.Add(1)
-		go watchEthStorage(&sw, &wg)
+		switch storageDiffsSource {
+		case "geth":
+			log.Debug("fetching storage diffs from geth pub sub")
+			rpcClient, _ := getClients()
+			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
+			payloadChan := make(chan statediff.Payload)
+			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		default:
+			log.Debug("fetching storage diffs from csv")
+			tailer := fs.FileTailer{Path: storageDiffsPath}
+			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		}
 	}
 
 	if len(ethContractInitializers) > 0 {
@@ -166,7 +182,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	}
 }
 
-func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
+func watchEthStorage(w watcher.IStorageWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the StorageTransformerInitializer set using the storage watcher
 	LogWithCommand.Info("executing storage transformers")
@@ -174,8 +190,8 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 	defer ticker.Stop()
 	for range ticker.C {
 		errs := make(chan error)
-		rows := make(chan storageUtils.StorageDiffRow)
-		w.Execute(rows, errs, queueRecheckInterval)
+		diffs := make(chan storageUtils.StorageDiff)
+		w.Execute(diffs, errs, queueRecheckInterval)
 	}
 }
 

cmd/headerSync.go

@@ -25,7 +25,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
@@ -100,7 +100,7 @@ func headerSync() {
 	}
 }
 
-func validateArgs(blockChain *geth.BlockChain) {
+func validateArgs(blockChain *eth.BlockChain) {
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
 		LogWithCommand.Error("validateArgs: Error getting last block: ", err)

cmd/root.go

@@ -28,10 +28,10 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 )
 
 var (
@@ -48,6 +48,7 @@ var (
 	recheckHeadersArg    bool
 	SubCommand           string
 	LogWithCommand       log.Entry
+	storageDiffsSource   string
 )
 
 const (
@@ -80,6 +81,7 @@ func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
 	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
+	storageDiffsSource = viper.GetString("storageDiffs.source")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -118,6 +120,7 @@ func init() {
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
+	rootCmd.PersistentFlags().String("storageDiffs-source", "csv", "where to get the state diffs: csv or geth")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
@@ -129,6 +132,7 @@ func init() {
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
+	viper.BindPFlag("storageDiffs.source", rootCmd.PersistentFlags().Lookup("storageDiffs-source"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
@@ -151,7 +155,15 @@ func initConfig() {
 	}
 }
 
-func getBlockChain() *geth.BlockChain {
+func getBlockChain() *eth.BlockChain {
+	rpcClient, ethClient := getClients()
+	vdbEthClient := client.NewEthClient(ethClient)
+	vdbNode := node.MakeNode(rpcClient)
+	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+	return eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+}
+
+func getClients() (client.RpcClient, *ethclient.Client) {
 	rawRpcClient, err := rpc.Dial(ipc)
 
 	if err != nil {
@@ -159,8 +171,6 @@ func getBlockChain() *geth.BlockChain {
 	}
 	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
 	ethClient := ethclient.NewClient(rawRpcClient)
-	vdbEthClient := client.NewEthClient(ethClient)
-	vdbNode := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+
+	return rpcClient, ethClient
 }

db/migrations/00023_create_headers_table.sql

@@ -8,7 +8,8 @@ CREATE TABLE public.headers
     block_timestamp      NUMERIC,
     check_count          INTEGER NOT NULL DEFAULT 0,
     eth_node_id          INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
-    eth_node_fingerprint VARCHAR(128)
+    eth_node_fingerprint VARCHAR(128),
+    UNIQUE (block_number, hash, eth_node_fingerprint)
 );
 
 -- Index is removed when table is

db/schema.sql

@@ -921,6 +921,14 @@ ALTER TABLE ONLY public.header_sync_transactions
     ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: headers headers_block_number_hash_eth_node_fingerprint_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.headers
+    ADD CONSTRAINT headers_block_number_hash_eth_node_fingerprint_key UNIQUE (block_number, hash, eth_node_fingerprint);
+
+
 --
 -- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --

go.mod

@@ -12,9 +12,11 @@ require (
 	github.com/ethereum/go-ethereum v1.9.5
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 // indirect
-	github.com/hashicorp/golang-lru v0.5.1
+	github.com/hashicorp/golang-lru v0.5.3
+	github.com/howeyc/fsnotify v0.9.0 // indirect
 	github.com/hpcloud/tail v1.0.0
 	github.com/huin/goupnp v1.0.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
@@ -42,8 +44,14 @@ require (
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.2 // indirect
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
+	golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 // indirect
 	golang.org/x/net v0.0.0-20190603091049-60506f45cf65
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
 )
+
+replace github.com/ethereum/go-ethereum => github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101
+
+replace gopkg.in/urfave/cli.v1 => gopkg.in/urfave/cli.v1 v1.20.0

go.sum

@@ -91,6 +91,8 @@ github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6/go.mod h1
 github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
@@ -241,6 +243,10 @@ github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2
 github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8=
 github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug=
 github.com/vulcanize/vulcanizedb v0.0.5/go.mod h1:utXkheCL9VjTfmuivuvRiAAyHh54GSN9XRQNEbFCA8k=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
@@ -252,6 +258,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 h1:4Yo9XtTfxfBCecLiBW8TYsFIdN7TkDhjGLWetFo4JSo=
+golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
@@ -281,6 +289,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aW
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -300,6 +310,7 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSv
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
 gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

integration_test/block_rewards_test.go

@@ -22,10 +22,10 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -39,7 +39,7 @@ var _ = Describe("Rewards calculations", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-		blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+		blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.Reward).To(Equal("5313550000000000000"))
@@ -53,7 +53,7 @@ var _ = Describe("Rewards calculations", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-		blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+		blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.UnclesReward).To(Equal("6875000000000000000"))

integration_test/contract_test.go

@@ -26,11 +26,11 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/testing"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/testing"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -56,7 +56,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			contract := testing.SampleContract()
 
 			logs, err := blockChain.GetFullSyncLogs(contract, big.NewInt(4703824), nil)
@@ -74,7 +74,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 			logs, err := blockChain.GetFullSyncLogs(core.Contract{Hash: "0x123"}, big.NewInt(4703824), nil)
 
@@ -92,7 +92,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 			contract := testing.SampleContract()
 			var balance = new(big.Int)

integration_test/contract_watcher_full_transformer_test.go

@@ -18,7 +18,6 @@ package integration
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"math/rand"
 	"strings"
 	"time"
@@ -27,6 +26,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

integration_test/contract_watcher_header_sync_transformer_test.go

@@ -40,8 +40,8 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 	var blockChain core.BlockChain
 	var headerRepository repositories.HeaderRepository
 	var headerID int64
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)   // 0x314159265dd8dbb310642f98f50c066173c1259b
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e
 
 	BeforeEach(func() {
 		db, blockChain = test_helpers.SetupDBandBC()
@@ -78,11 +78,10 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 			Expect(c.Address).To(Equal(tusdAddr))
 		})
 
-		It("Fails to initialize if first and block cannot be fetched from vDB headers table", func() {
+		It("initializes when no headers available in db", func() {
 			t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db)
 			err = t.Init()
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
+			Expect(err).ToNot(HaveOccurred())
 		})
 
 		It("Does nothing if nothing if no addresses are configured", func() {
@@ -378,6 +377,42 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 			Expect(transferLog.Value).To(Equal("2800000000000000000000"))
 		})
 
+		It("Marks header checked for a contract that has no logs at that header", func() {
+			t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t.Start).To(Equal(int64(6885702)))
+
+			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			Expect(err).ToNot(HaveOccurred())
+			transferLog := test_helpers.HeaderSyncTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID))
+
+			type checkedHeader struct {
+				ID       int64 `db:"id"`
+				HeaderID int64 `db:"header_id"`
+				NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"`
+				Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"`
+			}
+
+			transferCheckedHeader := new(checkedHeader)
+			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(transferCheckedHeader.Transfer).To(Equal(int64(1)))
+			Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1)))
+
+			newOwnerCheckedHeader := new(checkedHeader)
+			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1)))
+			Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1)))
+		})
+
 		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
 			var testConf config.ContractConfig
 			testConf = test_helpers.ENSandTusdConfig

integration_test/geth_blockchain_test.go

@@ -24,17 +24,17 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Reading from the Geth blockchain", func() {
-	var blockChain *geth.BlockChain
+	var blockChain *eth.BlockChain
 
 	BeforeEach(func() {
 		rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
@@ -44,7 +44,7 @@ var _ = Describe("Reading from the Geth blockchain", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-		blockChain = geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+		blockChain = eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 	})
 
 	It("reads two blocks", func(done Done) {

integration_test/integration_test_suite_test.go

@@ -17,6 +17,8 @@
 package integration_test
 
 import (
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -27,3 +29,7 @@ func TestIntegrationTest(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "IntegrationTest Suite")
 }
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

libraries/shared/factories/event/converter.go

@@ -16,9 +16,13 @@
 
 package event
 
-import "github.com/vulcanize/vulcanizedb/pkg/core"
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
 
+// Converter transforms log data into general InsertionModels the Repository can persist__
 type Converter interface {
-	ToEntities(contractAbi string, ethLog []core.HeaderSyncLog) ([]interface{}, error)
-	ToModels([]interface{}) ([]interface{}, error)
+	ToModels(contractAbi string, ethLog []core.HeaderSyncLog) ([]InsertionModel, error)
+	SetDB(db *postgres.DB)
 }

libraries/shared/factories/event/repository.go

@@ -16,9 +16,163 @@
 
 package event
 
-import "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+import (
+	"database/sql/driver"
+	"fmt"
+	"github.com/vulcanize/vulcanizedb/utils"
+	"strings"
 
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+const SetLogTransformedQuery = `UPDATE public.header_sync_logs SET transformed = true WHERE id = $1`
+
+// Repository persists transformed values to the DB
 type Repository interface {
-	Create(models []interface{}) error
+	Create(models []InsertionModel) error
 	SetDB(db *postgres.DB)
 }
+
+// LogFK is the name of log foreign key columns
+const LogFK ColumnName = "log_id"
+
+// AddressFK is the name of address foreign key columns
+const AddressFK ColumnName = "address_id"
+
+// HeaderFK is the name of header foreign key columns
+const HeaderFK ColumnName = "header_id"
+
+// SchemaName is the schema to work with
+type SchemaName string
+
+// TableName identifies the table for inserting the data
+type TableName string
+
+// ColumnName identifies columns on the given table
+type ColumnName string
+
+// ColumnValues maps a column to the value for insertion. This is restricted to []byte, bool, float64, int64, string, time.Time
+type ColumnValues map[ColumnName]interface{}
+
+// ErrUnsupportedValue is thrown when a model supplies a type of value the postgres driver cannot handle.
+var ErrUnsupportedValue = func(value interface{}) error {
+	return fmt.Errorf("unsupported type of value supplied in model: %v (%T)", value, value)
+}
+
+// InsertionModel is the generalised data structure a converter returns, and contains everything the repository needs to
+// persist the converted data.
+type InsertionModel struct {
+	SchemaName     SchemaName
+	TableName      TableName
+	OrderedColumns []ColumnName // Defines the fields to insert, and in which order the table expects them
+	ColumnValues   ColumnValues // Associated values for columns, restricted to []byte, bool, float64, int64, string, time.Time
+}
+
+// ModelToQuery stores memoised insertion queries to minimise computation
+var ModelToQuery = map[string]string{}
+
+// GetMemoizedQuery gets/creates a DB insertion query for the model
+func GetMemoizedQuery(model InsertionModel) string {
+	// The schema and table name uniquely determines the insertion query, use that for memoization
+	queryKey := string(model.SchemaName) + string(model.TableName)
+	query, queryMemoized := ModelToQuery[queryKey]
+	if !queryMemoized {
+		query = GenerateInsertionQuery(model)
+		ModelToQuery[queryKey] = query
+	}
+	return query
+}
+
+// GenerateInsertionQuery creates an SQL insertion query from an insertion model.
+// Should be called through GetMemoizedQuery, so the query is not generated on each call to Create.
+func GenerateInsertionQuery(model InsertionModel) string {
+	var valuePlaceholders []string
+	var updateOnConflict []string
+	for i := 0; i < len(model.OrderedColumns); i++ {
+		valuePlaceholder := fmt.Sprintf("$%d", 1+i)
+		valuePlaceholders = append(valuePlaceholders, valuePlaceholder)
+		updateOnConflict = append(updateOnConflict,
+			fmt.Sprintf("%s = %s", model.OrderedColumns[i], valuePlaceholder))
+	}
+
+	baseQuery := `INSERT INTO %v.%v (%v) VALUES(%v)
+		ON CONFLICT (header_id, log_id) DO UPDATE SET %v;`
+
+	return fmt.Sprintf(baseQuery,
+		model.SchemaName,
+		model.TableName,
+		joinOrderedColumns(model.OrderedColumns),
+		strings.Join(valuePlaceholders, ", "),
+		strings.Join(updateOnConflict, ", "))
+}
+
+/*
+Create generates an insertion query and persists to the DB, given a slice of InsertionModels.
+ColumnValues are restricted to []byte, bool, float64, int64, string, time.Time.
+
+testModel = shared.InsertionModel{
+	SchemaName:     "public"
+	TableName:      "testEvent",
+	OrderedColumns: []string{"header_id", "log_id", "variable1"},
+	ColumnValues: ColumnValues{
+		"header_id": 303
+		"log_id":   "808",
+		"variable1": "value1",
+	},
+}
+*/
+func Create(models []InsertionModel, db *postgres.DB) error {
+	if len(models) == 0 {
+		return fmt.Errorf("repository got empty model slice")
+	}
+
+	tx, dbErr := db.Beginx()
+	if dbErr != nil {
+		return dbErr
+	}
+
+	for _, model := range models {
+		// Maps can't be iterated over in a reliable manner, so we rely on OrderedColumns to define the order to insert
+		// tx.Exec is variadically typed in the args, so if we wrap in []interface{} we can apply them all automatically
+		var args []interface{}
+		for _, col := range model.OrderedColumns {
+			value := model.ColumnValues[col]
+			// Check whether or not PG can accept the type of value in the model
+			okPgValue := driver.IsValue(value)
+			if !okPgValue {
+				logrus.WithField("model", model).Errorf("PG cannot handle value of this type: %T", value)
+				return ErrUnsupportedValue(value)
+			}
+			args = append(args, value)
+		}
+
+		insertionQuery := GetMemoizedQuery(model)
+		_, execErr := tx.Exec(insertionQuery, args...) // couldn't pass varying types in bulk with args :: []string
+
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Error("failed to rollback ", rollbackErr)
+			}
+			return execErr
+		}
+
+		_, logErr := tx.Exec(SetLogTransformedQuery, model.ColumnValues[LogFK])
+
+		if logErr != nil {
+			utils.RollbackAndLogFailure(tx, logErr, "header_sync_logs.transformed")
+			return logErr
+		}
+	}
+
+	return tx.Commit()
+}
+
+func joinOrderedColumns(columns []ColumnName) string {
+	var stringColumns []string
+	for _, columnName := range columns {
+		stringColumns = append(stringColumns, string(columnName))
+	}
+	return strings.Join(stringColumns, ", ")
+}

libraries/shared/factories/event/repository_test.go

@@ -0,0 +1,204 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package event_test
+
+import (
+	"fmt"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+	"math/big"
+)
+
+var _ = Describe("Repository", func() {
+	var db *postgres.DB
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+	})
+
+	Describe("Create", func() {
+		const createTestEventTableQuery = `CREATE TABLE public.testEvent(
+		id        SERIAL PRIMARY KEY,
+		header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+		log_id    BIGINT  NOT NULL REFERENCES header_sync_logs (id) ON DELETE CASCADE,
+		variable1 TEXT,
+		UNIQUE (header_id, log_id)
+		);`
+
+		var (
+			headerID, logID  int64
+			headerRepository repositories.HeaderRepository
+			testModel        event.InsertionModel
+		)
+
+		BeforeEach(func() {
+			_, tableErr := db.Exec(createTestEventTableQuery)
+			Expect(tableErr).NotTo(HaveOccurred())
+			headerRepository = repositories.NewHeaderRepository(db)
+			var insertHeaderErr error
+			headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(insertHeaderErr).NotTo(HaveOccurred())
+			headerSyncLog := test_data.CreateTestLog(headerID, db)
+			logID = headerSyncLog.ID
+
+			testModel = event.InsertionModel{
+				SchemaName: "public",
+				TableName:  "testEvent",
+				OrderedColumns: []event.ColumnName{
+					event.HeaderFK, event.LogFK, "variable1",
+				},
+				ColumnValues: event.ColumnValues{
+					event.HeaderFK: headerID,
+					event.LogFK:    logID,
+					"variable1":    "value1",
+				},
+			}
+		})
+
+		AfterEach(func() {
+			db.MustExec(`DROP TABLE public.testEvent;`)
+		})
+
+		// Needs to run before the other tests, since those insert keys in map
+		It("memoizes queries", func() {
+			Expect(len(event.ModelToQuery)).To(Equal(0))
+			event.GetMemoizedQuery(testModel)
+			Expect(len(event.ModelToQuery)).To(Equal(1))
+			event.GetMemoizedQuery(testModel)
+			Expect(len(event.ModelToQuery)).To(Equal(1))
+		})
+
+		It("persists a model to postgres", func() {
+			createErr := event.Create([]event.InsertionModel{testModel}, db)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var res TestEvent
+			dbErr := db.Get(&res, `SELECT log_id, variable1 FROM public.testEvent;`)
+			Expect(dbErr).NotTo(HaveOccurred())
+
+			Expect(res.LogID).To(Equal(fmt.Sprint(testModel.ColumnValues[event.LogFK])))
+			Expect(res.Variable1).To(Equal(testModel.ColumnValues["variable1"]))
+		})
+
+		Describe("returns errors", func() {
+			It("for empty model slice", func() {
+				err := event.Create([]event.InsertionModel{}, db)
+				Expect(err).To(MatchError("repository got empty model slice"))
+			})
+
+			It("for failed SQL inserts", func() {
+				header := fakes.GetFakeHeader(1)
+				headerID, headerErr := headerRepository.CreateOrUpdateHeader(header)
+				Expect(headerErr).NotTo(HaveOccurred())
+
+				brokenModel := event.InsertionModel{
+					SchemaName: "public",
+					TableName:  "testEvent",
+					// Wrong name of last column compared to DB, will generate incorrect query
+					OrderedColumns: []event.ColumnName{
+						event.HeaderFK, event.LogFK, "variable2",
+					},
+					ColumnValues: event.ColumnValues{
+						event.HeaderFK: headerID,
+						event.LogFK:    logID,
+						"variable1":    "value1",
+					},
+				}
+
+				// Remove cached queries, or we won't generate a new (incorrect) one
+				delete(event.ModelToQuery, "publictestEvent")
+
+				createErr := event.Create([]event.InsertionModel{brokenModel}, db)
+				// Remove incorrect query, so other tests won't get it
+				delete(event.ModelToQuery, "publictestEvent")
+
+				Expect(createErr).To(HaveOccurred())
+			})
+
+			It("for unsupported types in ColumnValue", func() {
+				unsupportedValue := big.NewInt(5)
+				testModel = event.InsertionModel{
+					SchemaName: "public",
+					TableName:  "testEvent",
+					OrderedColumns: []event.ColumnName{
+						event.HeaderFK, event.LogFK, "variable1",
+					},
+					ColumnValues: event.ColumnValues{
+						event.HeaderFK: headerID,
+						event.LogFK:    logID,
+						"variable1":    unsupportedValue,
+					},
+				}
+
+				createErr := event.Create([]event.InsertionModel{testModel}, db)
+				Expect(createErr).To(MatchError(event.ErrUnsupportedValue(unsupportedValue)))
+			})
+		})
+
+		It("upserts queries with conflicting source", func() {
+			conflictingModel := event.InsertionModel{
+				SchemaName: "public",
+				TableName:  "testEvent",
+				OrderedColumns: []event.ColumnName{
+					event.HeaderFK, event.LogFK, "variable1",
+				},
+				ColumnValues: event.ColumnValues{
+					event.HeaderFK: headerID,
+					event.LogFK:    logID,
+					"variable1":    "conflictingValue",
+				},
+			}
+
+			createErr := event.Create([]event.InsertionModel{testModel, conflictingModel}, db)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var res TestEvent
+			dbErr := db.Get(&res, `SELECT log_id, variable1 FROM public.testEvent;`)
+			Expect(dbErr).NotTo(HaveOccurred())
+			Expect(res.Variable1).To(Equal(conflictingModel.ColumnValues["variable1"]))
+		})
+
+		It("generates correct queries", func() {
+			actualQuery := event.GenerateInsertionQuery(testModel)
+			expectedQuery := `INSERT INTO public.testEvent (header_id, log_id, variable1) VALUES($1, $2, $3)
+		ON CONFLICT (header_id, log_id) DO UPDATE SET header_id = $1, log_id = $2, variable1 = $3;`
+			Expect(actualQuery).To(Equal(expectedQuery))
+		})
+
+		It("marks log transformed", func() {
+			createErr := event.Create([]event.InsertionModel{testModel}, db)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var logTransformed bool
+			getErr := db.Get(&logTransformed, `SELECT transformed FROM public.header_sync_logs WHERE id = $1`, logID)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(logTransformed).To(BeTrue())
+		})
+	})
+})
+
+type TestEvent struct {
+	LogID     string `db:"log_id"`
+	Variable1 string
+}

libraries/shared/factories/event/transformer.go

@@ -30,6 +30,7 @@ type Transformer struct {
 }
 
 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.EventTransformer {
+	transformer.Converter.SetDB(db)
 	transformer.Repository.SetDB(db)
 	return transformer
 }
@@ -42,13 +43,7 @@ func (transformer Transformer) Execute(logs []core.HeaderSyncLog) error {
 		return nil
 	}
 
-	entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
-	if err != nil {
-		logrus.Errorf("error converting logs to entities in %v: %v", transformerName, err)
-		return err
-	}
-
-	models, err := transformer.Converter.ToModels(entities)
+	models, err := transformer.Converter.ToModels(config.ContractAbi, logs)
 	if err != nil {
 		logrus.Errorf("error converting entities to models in %v: %v", transformerName, err)
 		return err

libraries/shared/factories/event/transformer_test.go

@@ -66,12 +66,11 @@ var _ = Describe("Transformer", func() {
 		err := t.Execute([]core.HeaderSyncLog{})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
 		Expect(converter.ToModelsCalledCounter).To(Equal(0))
 		Expect(repository.CreateCalledCounter).To(Equal(0))
 	})
 
-	It("converts an eth log to an entity", func() {
+	It("converts an eth log to a model", func() {
 		err := t.Execute(logs)
 
 		Expect(err).NotTo(HaveOccurred())
@@ -79,26 +78,7 @@ var _ = Describe("Transformer", func() {
 		Expect(converter.LogsToConvert).To(Equal(logs))
 	})
 
-	It("returns an error if converter fails", func() {
-		converter.ToEntitiesError = fakes.FakeError
-
-		err := t.Execute(logs)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("converts an entity to a model", func() {
-		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
-
-		err := t.Execute(logs)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
-	})
-
 	It("returns an error if converting to models fails", func() {
-		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
 		converter.ToModelsError = fakes.FakeError
 
 		err := t.Execute(logs)
@@ -108,12 +88,12 @@ var _ = Describe("Transformer", func() {
 	})
 
 	It("persists the record", func() {
-		converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
+		converter.ModelsToReturn = []event.InsertionModel{test_data.GenericModel}
 
 		err := t.Execute(logs)
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel{}))
+		Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel))
 	})
 
 	It("returns error if persisting the record fails", func() {

libraries/shared/factories/storage/README.md

@@ -31,15 +31,15 @@ The storage transformer depends on contract-specific implementations of code cap
 
 ```golang
 func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
-	value, decodeErr := shared.Decode(row, metadata)
+	value, decodeErr := utils.Decode(diff, metadata)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
 }
 ```
 
@@ -47,20 +47,36 @@ func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
 
 In order to watch an additional smart contract, a developer must create three things:
 
-1. Mappings - specify how to identify keys in the contract's storage trie.
+1. StorageKeysLoader - identify keys in the contract's storage trie, providing metadata to describe how associated values should be decoded.
 1. Repository - specify how to persist a parsed version of the storage value matching the recognized storage key.
 1. Instance - create an instance of the storage transformer that uses your mappings and repository.
 
-### Mappings
+### StorageKeysLoader
+
+A `StorageKeysLoader` is used by the `StorageKeysLookup` object on a storage transformer.
 
 ```golang
-type Mappings interface {
-	Lookup(key common.Hash) (shared.StorageValueMetadata, error)
+type KeysLoader interface {
+	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
 	SetDB(db *postgres.DB)
 }
 ```
 
-A contract-specific implementation of the mappings interface enables the storage transformer to fetch metadata associated with a storage key.
+When a key is not found, the lookup object refreshes its known keys by calling the loader.
+
+```golang
+func (lookup *keysLookup) refreshMappings() error {
+	var err error
+	lookup.mappings, err = lookup.loader.LoadMappings()
+	if err != nil {
+		return err
+	}
+	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
+	return nil
+}
+```
+
+A contract-specific implementation of the loader enables the storage transformer to fetch metadata associated with a storage key.
 
 Storage metadata contains: the name of the variable matching the storage key, a raw version of any keys associated with the variable (if the variable is a mapping), and the variable's type.
 
@@ -72,7 +88,7 @@ type StorageValueMetadata struct {
 }
 ```
 
-Keys are only relevant if the variable is a mapping. For example, in the following Solidity code:
+The `Keys` field on the metadata is only relevant if the variable is a mapping. For example, in the following Solidity code:
 
 ```solidity
 pragma solidity ^0.4.0;
@@ -85,7 +101,7 @@ contract Contract {
 
 The metadata for variable `x` would not have any associated keys, but the metadata for a storage key associated with `y` would include the address used to specify that key's index in the mapping.
 
-The `SetDB` function is required for the mappings to connect to the database.
+The `SetDB` function is required for the storage key loader to connect to the database.
 A database connection may be desired when keys in a mapping variable need to be read from log events (e.g. to lookup what addresses may exist in `y`, above).
 
 ### Repository

libraries/shared/factories/storage/keys_loader.go

@@ -14,14 +14,15 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package repository
+package storage
 
-import "github.com/jmoiron/sqlx"
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
 
-func MarkContractWatcherHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
-	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
-		VALUES ($1, $2)
-		ON CONFLICT (header_id) DO
-		UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
-	return err
+type KeysLoader interface {
+	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
 }

libraries/shared/factories/storage/keys_lookup.go

@@ -0,0 +1,66 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type KeysLookup interface {
+	Lookup(key common.Hash) (utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}
+
+type keysLookup struct {
+	loader   KeysLoader
+	mappings map[common.Hash]utils.StorageValueMetadata
+}
+
+func NewKeysLookup(loader KeysLoader) KeysLookup {
+	return &keysLookup{loader: loader, mappings: make(map[common.Hash]utils.StorageValueMetadata)}
+}
+
+func (lookup *keysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
+	metadata, ok := lookup.mappings[key]
+	if !ok {
+		refreshErr := lookup.refreshMappings()
+		if refreshErr != nil {
+			return metadata, refreshErr
+		}
+		metadata, ok = lookup.mappings[key]
+		if !ok {
+			return metadata, utils.ErrStorageKeyNotFound{Key: key.Hex()}
+		}
+	}
+	return metadata, nil
+}
+
+func (lookup *keysLookup) refreshMappings() error {
+	var err error
+	lookup.mappings, err = lookup.loader.LoadMappings()
+	if err != nil {
+		return err
+	}
+	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
+	return nil
+}
+
+func (lookup *keysLookup) SetDB(db *postgres.DB) {
+	lookup.loader.SetDB(db)
+}

libraries/shared/factories/storage/keys_lookup_test.go

@@ -0,0 +1,113 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Storage keys lookup", func() {
+	var (
+		fakeMetadata = utils.GetStorageValueMetadata("name", map[utils.Key]string{}, utils.Uint256)
+		lookup       storage.KeysLookup
+		loader       *mocks.MockStorageKeysLoader
+	)
+
+	BeforeEach(func() {
+		loader = &mocks.MockStorageKeysLoader{}
+		lookup = storage.NewKeysLookup(loader)
+	})
+
+	Describe("Lookup", func() {
+		Describe("when key not found", func() {
+			It("refreshes keys", func() {
+				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+
+			It("returns error if refreshing keys fails", func() {
+				loader.LoadMappingsError = fakes.FakeError
+
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+		})
+
+		Describe("when key found", func() {
+			BeforeEach(func() {
+				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+				_, err := lookup.Lookup(fakes.FakeHash)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+
+			It("does not refresh keys", func() {
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+		})
+
+		It("returns metadata for loaded static key", func() {
+			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+
+			metadata, err := lookup.Lookup(fakes.FakeHash)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(metadata).To(Equal(fakeMetadata))
+		})
+
+		It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() {
+			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+
+			hashedKey := common.BytesToHash(crypto.Keccak256(fakes.FakeHash.Bytes()))
+			metadata, err := lookup.Lookup(hashedKey)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(metadata).To(Equal(fakeMetadata))
+		})
+
+		It("returns key not found error if key not found", func() {
+			_, err := lookup.Lookup(fakes.FakeHash)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(utils.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
+		})
+	})
+
+	Describe("SetDB", func() {
+		It("sets the db on the loader", func() {
+			lookup.SetDB(test_config.NewTestDB(test_config.NewTestNode()))
+
+			Expect(loader.SetDBCalled).To(BeTrue())
+		})
+	})
+})

libraries/shared/factories/storage/transformer.go

@@ -18,37 +18,35 @@ package storage
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type Transformer struct {
-	Address    common.Address
-	Mappings   storage.Mappings
-	Repository Repository
+	HashedAddress     common.Hash
+	StorageKeysLookup KeysLookup
+	Repository        Repository
 }
 
 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
-	transformer.Mappings.SetDB(db)
+	transformer.StorageKeysLookup.SetDB(db)
 	transformer.Repository.SetDB(db)
 	return transformer
 }
 
-func (transformer Transformer) ContractAddress() common.Address {
-	return transformer.Address
+func (transformer Transformer) KeccakContractAddress() common.Hash {
+	return transformer.HashedAddress
 }
 
-func (transformer Transformer) Execute(row utils.StorageDiffRow) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+func (transformer Transformer) Execute(diff utils.StorageDiff) error {
+	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
-	value, decodeErr := utils.Decode(row, metadata)
+	value, decodeErr := utils.Decode(diff, metadata)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
 }

libraries/shared/factories/storage/transformer_test.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
@@ -29,38 +28,38 @@ import (
 
 var _ = Describe("Storage transformer", func() {
 	var (
-		mappings   *mocks.MockMappings
-		repository *mocks.MockStorageRepository
-		t          storage.Transformer
+		storageKeysLookup *mocks.MockStorageKeysLookup
+		repository        *mocks.MockStorageRepository
+		t                 storage.Transformer
 	)
 
 	BeforeEach(func() {
-		mappings = &mocks.MockMappings{}
+		storageKeysLookup = &mocks.MockStorageKeysLookup{}
 		repository = &mocks.MockStorageRepository{}
 		t = storage.Transformer{
-			Address:    common.Address{},
-			Mappings:   mappings,
-			Repository: repository,
+			HashedAddress:     common.Hash{},
+			StorageKeysLookup: storageKeysLookup,
+			Repository:        repository,
 		}
 	})
 
 	It("returns the contract address being watched", func() {
-		fakeAddress := common.HexToAddress("0x12345")
-		t.Address = fakeAddress
+		fakeAddress := utils.HexToKeccak256Hash("0x12345")
+		t.HashedAddress = fakeAddress
 
-		Expect(t.ContractAddress()).To(Equal(fakeAddress))
+		Expect(t.KeccakContractAddress()).To(Equal(fakeAddress))
 	})
 
 	It("looks up metadata for storage key", func() {
-		t.Execute(utils.StorageDiffRow{})
+		t.Execute(utils.StorageDiff{})
 
-		Expect(mappings.LookupCalled).To(BeTrue())
+		Expect(storageKeysLookup.LookupCalled).To(BeTrue())
 	})
 
 	It("returns error if lookup fails", func() {
-		mappings.LookupErr = fakes.FakeError
+		storageKeysLookup.LookupErr = fakes.FakeError
 
-		err := t.Execute(utils.StorageDiffRow{})
+		err := t.Execute(utils.StorageDiff{})
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -68,16 +67,16 @@ var _ = Describe("Storage transformer", func() {
 
 	It("creates storage row with decoded data", func() {
 		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		mappings.Metadata = fakeMetadata
+		storageKeysLookup.Metadata = fakeMetadata
 		rawValue := common.HexToAddress("0x12345")
 		fakeBlockNumber := 123
 		fakeBlockHash := "0x67890"
-		fakeRow := utils.StorageDiffRow{
-			Contract:     common.Address{},
-			BlockHash:    common.HexToHash(fakeBlockHash),
-			BlockHeight:  fakeBlockNumber,
-			StorageKey:   common.Hash{},
-			StorageValue: rawValue.Hash(),
+		fakeRow := utils.StorageDiff{
+			HashedAddress: common.Hash{},
+			BlockHash:     common.HexToHash(fakeBlockHash),
+			BlockHeight:   fakeBlockNumber,
+			StorageKey:    common.Hash{},
+			StorageValue:  rawValue.Hash(),
 		}
 
 		err := t.Execute(fakeRow)
@@ -92,10 +91,10 @@ var _ = Describe("Storage transformer", func() {
 	It("returns error if creating row fails", func() {
 		rawValue := common.HexToAddress("0x12345")
 		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		mappings.Metadata = fakeMetadata
+		storageKeysLookup.Metadata = fakeMetadata
 		repository.CreateErr = fakes.FakeError
 
-		err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+		err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -119,13 +118,13 @@ var _ = Describe("Storage transformer", func() {
 		}
 
 		It("passes the decoded data items to the repository", func() {
-			mappings.Metadata = fakeMetadata
-			fakeRow := utils.StorageDiffRow{
-				Contract:     common.Address{},
-				BlockHash:    common.HexToHash(fakeBlockHash),
-				BlockHeight:  fakeBlockNumber,
-				StorageKey:   common.Hash{},
-				StorageValue: rawValue.Hash(),
+			storageKeysLookup.Metadata = fakeMetadata
+			fakeRow := utils.StorageDiff{
+				HashedAddress: common.Hash{},
+				BlockHash:     common.HexToHash(fakeBlockHash),
+				BlockHeight:   fakeBlockNumber,
+				StorageKey:    common.Hash{},
+				StorageValue:  rawValue.Hash(),
 			}
 
 			err := t.Execute(fakeRow)
@@ -141,10 +140,10 @@ var _ = Describe("Storage transformer", func() {
 		})
 
 		It("returns error if creating a row fails", func() {
-			mappings.Metadata = fakeMetadata
+			storageKeysLookup.Metadata = fakeMetadata
 			repository.CreateErr = fakes.FakeError
 
-			err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+			err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))

libraries/shared/fetcher/csv_tail_storage_fetcher.go

@@ -17,18 +17,12 @@
 package fetcher
 
 import (
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
+	"strings"
 )
 
-type IStorageFetcher interface {
-	FetchStorageDiffs(chan<- utils.StorageDiffRow, chan<- error)
-}
-
 type CsvTailStorageFetcher struct {
 	tailer fs.Tailer
 }
@@ -37,18 +31,18 @@ func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
 	return CsvTailStorageFetcher{tailer: tailer}
 }
 
-func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	t, tailErr := storageFetcher.tailer.Tail()
 	if tailErr != nil {
 		errs <- tailErr
 	}
-	log.Debug("fetching storage diffs...")
+	logrus.Debug("fetching storage diffs...")
 	for line := range t.Lines {
-		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+		diff, parseErr := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		if parseErr != nil {
 			errs <- parseErr
 		} else {
-			out <- row
+			out <- diff
 		}
 	}
 }

libraries/shared/fetcher/csv_tail_storage_fetcher_test.go

@@ -18,30 +18,28 @@ package fetcher_test
 
 import (
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/hpcloud/tail"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"strings"
+	"time"
 )
 
 var _ = Describe("Csv Tail Storage Fetcher", func() {
 	var (
 		errorsChannel  chan error
 		mockTailer     *fakes.MockTailer
-		rowsChannel    chan utils.StorageDiffRow
+		diffsChannel   chan utils.StorageDiff
 		storageFetcher fetcher.CsvTailStorageFetcher
 	)
 
 	BeforeEach(func() {
 		errorsChannel = make(chan error)
-		rowsChannel = make(chan utils.StorageDiffRow)
+		diffsChannel = make(chan utils.StorageDiff)
 		mockTailer = fakes.NewMockTailer()
 		storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
 	})
@@ -49,7 +47,7 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 	It("adds error to errors channel if tailing file fails", func(done Done) {
 		mockTailer.TailErr = fakes.FakeError
 
-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 
 		Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
 		close(done)
@@ -58,24 +56,24 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 	It("adds parsed csv row to rows channel for storage diff", func(done Done) {
 		line := getFakeLine()
 
-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line
 
-		expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
+		expectedRow, err := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		Expect(err).NotTo(HaveOccurred())
-		Expect(<-rowsChannel).To(Equal(expectedRow))
+		Expect(<-diffsChannel).To(Equal(expectedRow))
 		close(done)
 	})
 
 	It("adds error to errors channel if parsing csv fails", func(done Done) {
 		line := &tail.Line{Text: "invalid"}
 
-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line
 
 		Expect(<-errorsChannel).To(HaveOccurred())
 		select {
-		case <-rowsChannel:
+		case <-diffsChannel:
 			Fail("value passed to rows channel on error")
 		default:
 			Succeed()

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -0,0 +1,81 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher
+
+import (
+	"fmt"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+)
+
+type GethRpcStorageFetcher struct {
+	statediffPayloadChan chan statediff.Payload
+	streamer             streamer.Streamer
+}
+
+func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan statediff.Payload) GethRpcStorageFetcher {
+	return GethRpcStorageFetcher{
+		statediffPayloadChan: statediffPayloadChan,
+		streamer:             streamer,
+	}
+}
+
+func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
+	ethStatediffPayloadChan := fetcher.statediffPayloadChan
+	clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
+	if clientSubErr != nil {
+		errs <- clientSubErr
+		panic(fmt.Sprintf("Error creating a geth client subscription: %v", clientSubErr))
+	}
+	logrus.Info("Successfully created a geth client subscription: ", clientSubscription)
+
+	for {
+		diff := <-ethStatediffPayloadChan
+		logrus.Trace("received a statediff")
+		stateDiff := new(statediff.StateDiff)
+		decodeErr := rlp.DecodeBytes(diff.StateDiffRlp, stateDiff)
+		if decodeErr != nil {
+			logrus.Warn("Error decoding state diff into RLP: ", decodeErr)
+			errs <- decodeErr
+		}
+
+		accounts := getAccountsFromDiff(*stateDiff)
+		logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
+		for _, account := range accounts {
+			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
+			for _, storage := range account.Storage {
+				diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
+				logrus.Trace("adding storage diff to out channel",
+					"keccak of address: ", diff.HashedAddress.Hex(),
+					"block height: ", diff.BlockHeight,
+					"storage key: ", diff.StorageKey.Hex(),
+					"storage value: ", diff.StorageValue.Hex())
+				if formatErr != nil {
+					errs <- formatErr
+				}
+
+				out <- diff
+			}
+		}
+	}
+}
+
+func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
+	accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
+	return append(accounts, stateDiff.DeletedAccounts...)
+}

libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go

@@ -0,0 +1,172 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+type MockStoragediffStreamer struct {
+	subscribeError    error
+	PassedPayloadChan chan statediff.Payload
+	streamPayloads    []statediff.Payload
+}
+
+func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	clientSubscription := rpc.ClientSubscription{}
+	streamer.PassedPayloadChan = statediffPayloadChan
+
+	go func() {
+		for _, payload := range streamer.streamPayloads {
+			streamer.PassedPayloadChan <- payload
+		}
+	}()
+
+	return &clientSubscription, streamer.subscribeError
+}
+
+func (streamer *MockStoragediffStreamer) SetSubscribeError(err error) {
+	streamer.subscribeError = err
+}
+
+func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payload) {
+	streamer.streamPayloads = payloads
+}
+
+var _ = Describe("Geth RPC Storage Fetcher", func() {
+	var streamer MockStoragediffStreamer
+	var statediffPayloadChan chan statediff.Payload
+	var statediffFetcher fetcher.GethRpcStorageFetcher
+	var storagediffChan chan utils.StorageDiff
+	var errorChan chan error
+
+	BeforeEach(func() {
+		streamer = MockStoragediffStreamer{}
+		statediffPayloadChan = make(chan statediff.Payload, 1)
+		statediffFetcher = fetcher.NewGethRpcStorageFetcher(&streamer, statediffPayloadChan)
+		storagediffChan = make(chan utils.StorageDiff)
+		errorChan = make(chan error)
+	})
+
+	It("adds errors to error channel if the RPC subscription fails and panics", func(done Done) {
+		streamer.SetSubscribeError(fakes.FakeError)
+
+		go func() {
+			failedSub := func() {
+				statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+			}
+			Expect(failedSub).To(Panic())
+		}()
+
+		Expect(<-errorChan).To(MatchError(fakes.FakeError))
+		close(done)
+	})
+
+	It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		streamedPayload := <-statediffPayloadChan
+		Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload))
+		Expect(streamer.PassedPayloadChan).To(Equal(statediffPayloadChan))
+		close(done)
+	})
+
+	It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) {
+		badStatediffPayload := statediff.Payload{}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("EOF"))
+
+		close(done)
+	})
+
+	It("adds parsed statediff payloads to the rows channel", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		height := test_data.BlockNumber
+		intHeight := int(height.Int64())
+		createdExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
+		}
+		updatedExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.LargeStorageValue),
+		}
+		deletedExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
+		}
+
+		createdStateDiff := <-storagediffChan
+		updatedStateDiff := <-storagediffChan
+		deletedStateDiff := <-storagediffChan
+
+		Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff))
+		Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff))
+		Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff))
+
+		close(done)
+	})
+
+	It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) {
+		accountDiffs := test_data.CreatedAccountDiffs
+		accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue}
+
+		stateDiff := statediff.StateDiff{
+			BlockNumber:     test_data.BlockNumber,
+			BlockHash:       common.HexToHash(test_data.BlockHash),
+			CreatedAccounts: accountDiffs,
+		}
+
+		stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+		Expect(err).NotTo(HaveOccurred())
+
+		badStatediffPayload := statediff.Payload{
+			StateDiffRlp: stateDiffRlp,
+		}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("rlp: input contains more than one value"))
+
+		close(done)
+	})
+})

libraries/shared/fetcher/storage_fetcher_interface.go

@@ -0,0 +1,21 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher
+
+import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+
+type IStorageFetcher interface {
+	FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error)
+}

libraries/shared/mocks/event_converter.go

@@ -16,41 +16,29 @@
 
 package mocks
 
-import "github.com/vulcanize/vulcanizedb/pkg/core"
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
 
 type MockConverter struct {
-	ToEntitiesError         error
-	PassedContractAddresses []string
 	ToModelsError           error
-	entityConverterError    error
-	modelConverterError     error
 	ContractAbi             string
 	LogsToConvert           []core.HeaderSyncLog
-	EntitiesToConvert       []interface{}
-	EntitiesToReturn        []interface{}
-	ModelsToReturn          []interface{}
-	ToEntitiesCalledCounter int
+	ModelsToReturn          []event.InsertionModel
+	PassedContractAddresses []string
+	SetDBCalled             bool
 	ToModelsCalledCounter   int
 }
 
-func (converter *MockConverter) ToEntities(contractAbi string, ethLogs []core.HeaderSyncLog) ([]interface{}, error) {
-	for _, log := range ethLogs {
-		converter.PassedContractAddresses = append(converter.PassedContractAddresses, log.Log.Address.Hex())
-	}
-	converter.ContractAbi = contractAbi
-	converter.LogsToConvert = ethLogs
-	return converter.EntitiesToReturn, converter.ToEntitiesError
-}
-
-func (converter *MockConverter) ToModels(entities []interface{}) ([]interface{}, error) {
-	converter.EntitiesToConvert = entities
+func (converter *MockConverter) ToModels(abi string, logs []core.HeaderSyncLog) ([]event.InsertionModel, error) {
+	converter.LogsToConvert = logs
+	converter.ContractAbi = abi
+	converter.ToModelsCalledCounter = converter.ToModelsCalledCounter + 1
 	return converter.ModelsToReturn, converter.ToModelsError
 }
 
-func (converter *MockConverter) SetToEntityConverterError(err error) {
-	converter.entityConverterError = err
-}
-
-func (converter *MockConverter) SetToModelConverterError(err error) {
-	converter.modelConverterError = err
+func (converter *MockConverter) SetDB(db *postgres.DB) {
+	converter.SetDBCalled = true
 }

libraries/shared/mocks/event_repository.go

@@ -17,17 +17,18 @@
 package mocks
 
 import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type MockEventRepository struct {
 	createError         error
-	PassedModels        []interface{}
+	PassedModels        []event.InsertionModel
 	SetDbCalled         bool
 	CreateCalledCounter int
 }
 
-func (repository *MockEventRepository) Create(models []interface{}) error {
+func (repository *MockEventRepository) Create(models []event.InsertionModel) error {
 	repository.PassedModels = models
 	repository.CreateCalledCounter++
 

libraries/shared/mocks/storage_fetcher.go

@@ -19,21 +19,21 @@ package mocks
 import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 
 type MockStorageFetcher struct {
-	RowsToReturn []utils.StorageDiffRow
-	ErrsToReturn []error
+	DiffsToReturn []utils.StorageDiff
+	ErrsToReturn  []error
 }
 
 func NewMockStorageFetcher() *MockStorageFetcher {
 	return &MockStorageFetcher{}
 }
 
-func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	defer close(out)
 	defer close(errs)
 	for _, err := range fetcher.ErrsToReturn {
 		errs <- err
 	}
-	for _, row := range fetcher.RowsToReturn {
-		out <- row
+	for _, diff := range fetcher.DiffsToReturn {
+		out <- diff
 	}
 }

libraries/shared/mocks/storage_keys_loader.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockStorageKeysLoader struct {
+	LoadMappingsCallCount int
+	LoadMappingsError     error
+	SetDBCalled           bool
+	StorageKeyMappings    map[common.Hash]utils.StorageValueMetadata
+}
+
+func (loader *MockStorageKeysLoader) LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error) {
+	loader.LoadMappingsCallCount++
+	return loader.StorageKeyMappings, loader.LoadMappingsError
+}
+
+func (loader *MockStorageKeysLoader) SetDB(db *postgres.DB) {
+	loader.SetDBCalled = true
+}

libraries/shared/mocks/storage_keys_lookup.go

@@ -18,22 +18,21 @@ package mocks
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-type MockMappings struct {
+type MockStorageKeysLookup struct {
 	Metadata     utils.StorageValueMetadata
 	LookupCalled bool
 	LookupErr    error
 }
 
-func (mappings *MockMappings) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
+func (mappings *MockStorageKeysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
 	mappings.LookupCalled = true
 	return mappings.Metadata, mappings.LookupErr
 }
 
-func (*MockMappings) SetDB(db *postgres.DB) {
+func (*MockStorageKeysLookup) SetDB(db *postgres.DB) {
 	panic("implement me")
 }

libraries/shared/mocks/storage_queue.go

@@ -23,16 +23,16 @@ import (
 type MockStorageQueue struct {
 	AddCalled      bool
 	AddError       error
-	AddPassedRow   utils.StorageDiffRow
+	AddPassedDiff  utils.StorageDiff
 	DeleteErr      error
 	DeletePassedId int
 	GetAllErr      error
-	RowsToReturn   []utils.StorageDiffRow
+	DiffsToReturn  []utils.StorageDiff
 }
 
-func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error {
+func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
 	queue.AddCalled = true
-	queue.AddPassedRow = row
+	queue.AddPassedDiff = diff
 	return queue.AddError
 }
 
@@ -41,6 +41,6 @@ func (queue *MockStorageQueue) Delete(id int) error {
 	return queue.DeleteErr
 }
 
-func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
-	return queue.RowsToReturn, queue.GetAllErr
+func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
+	return queue.DiffsToReturn, queue.GetAllErr
 }

libraries/shared/mocks/storage_transformer.go

@@ -18,25 +18,24 @@ package mocks
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type MockStorageTransformer struct {
-	Address    common.Address
-	ExecuteErr error
-	PassedRow  utils.StorageDiffRow
+	KeccakOfAddress common.Hash
+	ExecuteErr      error
+	PassedDiff      utils.StorageDiff
 }
 
-func (transformer *MockStorageTransformer) Execute(row utils.StorageDiffRow) error {
-	transformer.PassedRow = row
+func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
+	transformer.PassedDiff = diff
 	return transformer.ExecuteErr
 }
 
-func (transformer *MockStorageTransformer) ContractAddress() common.Address {
-	return transformer.Address
+func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash {
+	return transformer.KeccakOfAddress
 }
 
 func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {

libraries/shared/repository/address_repository.go

@@ -0,0 +1,73 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/jmoiron/sqlx"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+const getOrCreateAddressQuery = `WITH addressId AS (
+			INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id
+		)
+		SELECT id FROM addresses WHERE address = $1
+		UNION
+		SELECT id FROM addressId`
+
+func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
+	checksumAddress := getChecksumAddress(address)
+
+	var addressId int64
+	getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
+
+	return addressId, getOrCreateErr
+}
+
+func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
+	checksumAddress := getChecksumAddress(address)
+
+	var addressId int64
+	getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
+
+	return addressId, getOrCreateErr
+}
+
+func GetAddressById(db *postgres.DB, id int64) (string, error) {
+	var address string
+	getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
+	return address, getErr
+}
+
+func getChecksumAddress(address string) string {
+	stringAddressToCommonAddress := common.HexToAddress(address)
+	return stringAddressToCommonAddress.Hex()
+}

libraries/shared/repository/address_repository_test.go

@@ -1,22 +1,23 @@
 // VulcanizeDB
 // Copyright © 2019 Vulcanize
-
+//
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
-
+//
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
-
+//
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package repositories_test
+package repository_test
 
 import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"strings"
 
 	"github.com/jmoiron/sqlx"
@@ -24,7 +25,6 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
@@ -32,13 +32,11 @@ import (
 var _ = Describe("address lookup", func() {
 	var (
 		db      *postgres.DB
-		repo    repositories.AddressRepository
 		address = fakes.FakeAddress.Hex()
 	)
 	BeforeEach(func() {
 		db = test_config.NewTestDB(test_config.NewTestNode())
 		test_config.CleanTestDB(db)
-		repo = repositories.AddressRepository{}
 	})
 
 	AfterEach(func() {
@@ -52,7 +50,7 @@ var _ = Describe("address lookup", func() {
 
 	Describe("GetOrCreateAddress", func() {
 		It("creates an address record", func() {
-			addressId, createErr := repo.GetOrCreateAddress(db, address)
+			addressId, createErr := repository.GetOrCreateAddress(db, address)
 			Expect(createErr).NotTo(HaveOccurred())
 
 			var actualAddress dbAddress
@@ -63,10 +61,10 @@ var _ = Describe("address lookup", func() {
 		})
 
 		It("returns the existing record id if the address already exists", func() {
-			createId, createErr := repo.GetOrCreateAddress(db, address)
+			createId, createErr := repository.GetOrCreateAddress(db, address)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			getId, getErr := repo.GetOrCreateAddress(db, address)
+			getId, getErr := repository.GetOrCreateAddress(db, address)
 			Expect(getErr).NotTo(HaveOccurred())
 
 			var addressCount int
@@ -78,20 +76,20 @@ var _ = Describe("address lookup", func() {
 
 		It("gets upper-cased addresses", func() {
 			upperAddress := strings.ToUpper(address)
-			upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddress(db, upperAddress)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
 		})
 
 		It("gets lower-cased addresses", func() {
 			lowerAddress := strings.ToLower(address)
-			upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddress(db, lowerAddress)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
 		})
@@ -112,7 +110,7 @@ var _ = Describe("address lookup", func() {
 		})
 
 		It("creates an address record", func() {
-			addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			addressId, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(createErr).NotTo(HaveOccurred())
 			commitErr := tx.Commit()
 			Expect(commitErr).NotTo(HaveOccurred())
@@ -125,10 +123,10 @@ var _ = Describe("address lookup", func() {
 		})
 
 		It("returns the existing record id if the address already exists", func() {
-			_, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			_, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			_, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			_, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			tx.Commit()
 
@@ -139,10 +137,10 @@ var _ = Describe("address lookup", func() {
 
 		It("gets upper-cased addresses", func() {
 			upperAddress := strings.ToUpper(address)
-			upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, upperAddress)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			tx.Commit()
 
@@ -151,10 +149,10 @@ var _ = Describe("address lookup", func() {
 
 		It("gets lower-cased addresses", func() {
 			lowerAddress := strings.ToLower(address)
-			upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, lowerAddress)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			tx.Commit()
 
@@ -164,16 +162,16 @@ var _ = Describe("address lookup", func() {
 
 	Describe("GetAddressById", func() {
 		It("gets and address by it's id", func() {
-			addressId, createErr := repo.GetOrCreateAddress(db, address)
+			addressId, createErr := repository.GetOrCreateAddress(db, address)
 			Expect(createErr).NotTo(HaveOccurred())
 
-			actualAddress, getErr := repo.GetAddressById(db, addressId)
+			actualAddress, getErr := repository.GetAddressById(db, addressId)
 			Expect(getErr).NotTo(HaveOccurred())
 			Expect(actualAddress).To(Equal(address))
 		})
 
 		It("returns an error if the id doesn't exist", func() {
-			_, getErr := repo.GetAddressById(db, 0)
+			_, getErr := repository.GetAddressById(db, 0)
 			Expect(getErr).To(HaveOccurred())
 			Expect(getErr).To(MatchError("sql: no rows in result set"))
 		})

libraries/shared/repository/repository_test.go

@@ -1,67 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("", func() {
-	Describe("MarkContractWatcherHeaderCheckedInTransaction", func() {
-		var (
-			checkedHeadersColumn string
-			db                   *postgres.DB
-		)
-
-		BeforeEach(func() {
-			db = test_config.NewTestDB(test_config.NewTestNode())
-			test_config.CleanTestDB(db)
-			checkedHeadersColumn = "test_column_checked"
-			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
-				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
-			Expect(migrateErr).NotTo(HaveOccurred())
-		})
-
-		AfterEach(func() {
-			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
-			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
-		})
-
-		It("marks passed header as checked within a passed transaction", func() {
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
-			Expect(headerErr).NotTo(HaveOccurred())
-			tx, txErr := db.Beginx()
-			Expect(txErr).NotTo(HaveOccurred())
-
-			err := repository.MarkContractWatcherHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
-			Expect(err).NotTo(HaveOccurred())
-			commitErr := tx.Commit()
-			Expect(commitErr).NotTo(HaveOccurred())
-			var checkedCount int
-			fetchErr := db.Get(&checkedCount, `SELECT COUNT(*) FROM public.checked_headers WHERE header_id = $1`, headerID)
-			Expect(fetchErr).NotTo(HaveOccurred())
-			Expect(checkedCount).To(Equal(1))
-		})
-	})
-})

libraries/shared/storage/storage_queue.go

@@ -22,9 +22,9 @@ import (
 )
 
 type IStorageQueue interface {
-	Add(row utils.StorageDiffRow) error
+	Add(diff utils.StorageDiff) error
 	Delete(id int) error
-	GetAll() ([]utils.StorageDiffRow, error)
+	GetAll() ([]utils.StorageDiff, error)
 }
 
 type StorageQueue struct {
@@ -35,11 +35,11 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
 	return StorageQueue{db: db}
 }
 
-func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
+func (queue StorageQueue) Add(diff utils.StorageDiff) error {
 	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
 		block_hash, block_height, storage_key, storage_value) VALUES
-		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, row.Contract.Bytes(), row.BlockHash.Bytes(),
-		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
+		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.HashedAddress.Bytes(), diff.BlockHash.Bytes(),
+		diff.BlockHeight, diff.StorageKey.Bytes(), diff.StorageValue.Bytes())
 	return err
 }
 
@@ -48,8 +48,8 @@ func (queue StorageQueue) Delete(id int) error {
 	return err
 }
 
-func (queue StorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
-	var result []utils.StorageDiffRow
+func (queue StorageQueue) GetAll() ([]utils.StorageDiff, error) {
+	var result []utils.StorageDiff
 	err := queue.db.Select(&result, `SELECT * FROM public.queued_storage`)
 	return result, err
 }

libraries/shared/storage/storage_queue_test.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -30,35 +29,36 @@ import (
 var _ = Describe("Storage queue", func() {
 	var (
 		db    *postgres.DB
-		row   utils.StorageDiffRow
+		diff  utils.StorageDiff
 		queue storage.IStorageQueue
 	)
 
 	BeforeEach(func() {
-		row = utils.StorageDiffRow{
-			Contract:     common.HexToAddress("0x123456"),
-			BlockHash:    common.HexToHash("0x678901"),
-			BlockHeight:  987,
-			StorageKey:   common.HexToHash("0x654321"),
-			StorageValue: common.HexToHash("0x198765"),
+		fakeAddr := "0x123456"
+		diff = utils.StorageDiff{
+			HashedAddress: utils.HexToKeccak256Hash(fakeAddr),
+			BlockHash:     common.HexToHash("0x678901"),
+			BlockHeight:   987,
+			StorageKey:    common.HexToHash("0x654321"),
+			StorageValue:  common.HexToHash("0x198765"),
 		}
 		db = test_config.NewTestDB(test_config.NewTestNode())
 		test_config.CleanTestDB(db)
 		queue = storage.NewStorageQueue(db)
-		addErr := queue.Add(row)
+		addErr := queue.Add(diff)
 		Expect(addErr).NotTo(HaveOccurred())
 	})
 
 	Describe("Add", func() {
-		It("adds a storage row to the db", func() {
-			var result utils.StorageDiffRow
+		It("adds a storage diff to the db", func() {
+			var result utils.StorageDiff
 			getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
 			Expect(getErr).NotTo(HaveOccurred())
-			Expect(result).To(Equal(row))
+			Expect(result).To(Equal(diff))
 		})
 
-		It("does not duplicate storage rows", func() {
-			addErr := queue.Add(row)
+		It("does not duplicate storage diffs", func() {
+			addErr := queue.Add(diff)
 			Expect(addErr).NotTo(HaveOccurred())
 			var count int
 			getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`)
@@ -67,12 +67,12 @@ var _ = Describe("Storage queue", func() {
 		})
 	})
 
-	It("deletes storage row from db", func() {
-		rows, getErr := queue.GetAll()
+	It("deletes storage diff from db", func() {
+		diffs, getErr := queue.GetAll()
 		Expect(getErr).NotTo(HaveOccurred())
-		Expect(len(rows)).To(Equal(1))
+		Expect(len(diffs)).To(Equal(1))
 
-		err := queue.Delete(rows[0].Id)
+		err := queue.Delete(diffs[0].Id)
 
 		Expect(err).NotTo(HaveOccurred())
 		remainingRows, secondGetErr := queue.GetAll()
@@ -80,33 +80,34 @@ var _ = Describe("Storage queue", func() {
 		Expect(len(remainingRows)).To(BeZero())
 	})
 
-	It("gets all storage rows from db", func() {
-		rowTwo := utils.StorageDiffRow{
-			Contract:     common.HexToAddress("0x123456"),
-			BlockHash:    common.HexToHash("0x678902"),
-			BlockHeight:  988,
-			StorageKey:   common.HexToHash("0x654322"),
-			StorageValue: common.HexToHash("0x198766"),
+	It("gets all storage diffs from db", func() {
+		fakeAddr := "0x234567"
+		diffTwo := utils.StorageDiff{
+			HashedAddress: utils.HexToKeccak256Hash(fakeAddr),
+			BlockHash:     common.HexToHash("0x678902"),
+			BlockHeight:   988,
+			StorageKey:    common.HexToHash("0x654322"),
+			StorageValue:  common.HexToHash("0x198766"),
 		}
-		addErr := queue.Add(rowTwo)
+		addErr := queue.Add(diffTwo)
 		Expect(addErr).NotTo(HaveOccurred())
 
-		rows, err := queue.GetAll()
+		diffs, err := queue.GetAll()
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(len(rows)).To(Equal(2))
-		Expect(rows[0]).NotTo(Equal(rows[1]))
-		Expect(rows[0].Id).NotTo(BeZero())
-		Expect(rows[0].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
-		Expect(rows[0].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
-		Expect(rows[0].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
-		Expect(rows[0].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
-		Expect(rows[0].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
-		Expect(rows[1].Id).NotTo(BeZero())
-		Expect(rows[1].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
-		Expect(rows[1].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
-		Expect(rows[1].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
-		Expect(rows[1].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
-		Expect(rows[1].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
+		Expect(len(diffs)).To(Equal(2))
+		Expect(diffs[0]).NotTo(Equal(diffs[1]))
+		Expect(diffs[0].Id).NotTo(BeZero())
+		Expect(diffs[0].HashedAddress).To(Or(Equal(diff.HashedAddress), Equal(diffTwo.HashedAddress)))
+		Expect(diffs[0].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
+		Expect(diffs[0].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
+		Expect(diffs[0].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
+		Expect(diffs[0].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
+		Expect(diffs[1].Id).NotTo(BeZero())
+		Expect(diffs[1].HashedAddress).To(Or(Equal(diff.HashedAddress), Equal(diffTwo.HashedAddress)))
+		Expect(diffs[1].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
+		Expect(diffs[1].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
+		Expect(diffs[1].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
+		Expect(diffs[1].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
 	})
 })

libraries/shared/storage/utils/decoder.go

@@ -27,20 +27,20 @@ const (
 	bitsPerByte = 8
 )
 
-func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
+func Decode(diff StorageDiff, metadata StorageValueMetadata) (interface{}, error) {
 	switch metadata.Type {
 	case Uint256:
-		return decodeInteger(row.StorageValue.Bytes()), nil
+		return decodeInteger(diff.StorageValue.Bytes()), nil
 	case Uint48:
-		return decodeInteger(row.StorageValue.Bytes()), nil
+		return decodeInteger(diff.StorageValue.Bytes()), nil
 	case Uint128:
-		return decodeInteger(row.StorageValue.Bytes()), nil
+		return decodeInteger(diff.StorageValue.Bytes()), nil
 	case Address:
-		return decodeAddress(row.StorageValue.Bytes()), nil
+		return decodeAddress(diff.StorageValue.Bytes()), nil
 	case Bytes32:
-		return row.StorageValue.Hex(), nil
+		return diff.StorageValue.Hex(), nil
 	case PackedSlot:
-		return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
+		return decodePackedSlot(diff.StorageValue.Bytes(), metadata.PackedTypes), nil
 	default:
 		panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
 	}

libraries/shared/storage/utils/decoder_test.go

@@ -29,10 +29,10 @@ import (
 var _ = Describe("Storage decoder", func() {
 	It("decodes uint256", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
-		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		diff := utils.StorageDiff{StorageValue: fakeInt}
 		metadata := utils.StorageValueMetadata{Type: utils.Uint256}
 
-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@@ -40,10 +40,10 @@ var _ = Describe("Storage decoder", func() {
 
 	It("decodes uint128", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
-		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		diff := utils.StorageDiff{StorageValue: fakeInt}
 		metadata := utils.StorageValueMetadata{Type: utils.Uint128}
 
-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@@ -51,10 +51,10 @@ var _ = Describe("Storage decoder", func() {
 
 	It("decodes uint48", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
-		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		diff := utils.StorageDiff{StorageValue: fakeInt}
 		metadata := utils.StorageValueMetadata{Type: utils.Uint48}
 
-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@@ -62,10 +62,10 @@ var _ = Describe("Storage decoder", func() {
 
 	It("decodes address", func() {
 		fakeAddress := common.HexToAddress("0x12345")
-		row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()}
+		diff := utils.StorageDiff{StorageValue: fakeAddress.Hash()}
 		metadata := utils.StorageValueMetadata{Type: utils.Address}
 
-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(fakeAddress.Hex()))
@@ -75,7 +75,7 @@ var _ = Describe("Storage decoder", func() {
 		It("decodes uint48 items", func() {
 			//this is a real storage data example
 			packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			diff := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Uint48
 			packedTypes[1] = utils.Uint48
@@ -85,7 +85,7 @@ var _ = Describe("Storage decoder", func() {
 				PackedTypes: packedTypes,
 			}
 
-			result, err := utils.Decode(row, metadata)
+			result, err := utils.Decode(diff, metadata)
 			decodedValues := result.(map[int]string)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -99,7 +99,7 @@ var _ = Describe("Storage decoder", func() {
 			packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
 
 			packedStorage := common.HexToHash(packedStorageHex)
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			diff := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Uint48
 			packedTypes[1] = utils.Uint48
@@ -112,7 +112,7 @@ var _ = Describe("Storage decoder", func() {
 				PackedTypes: packedTypes,
 			}
 
-			result, err := utils.Decode(row, metadata)
+			result, err := utils.Decode(diff, metadata)
 			decodedValues := result.(map[int]string)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -129,7 +129,7 @@ var _ = Describe("Storage decoder", func() {
 			packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
 				"0000000000000000AB54A98CEB1F0AD2"
 			packedStorage := common.HexToHash(packedStorageHex)
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			diff := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Uint128
 			packedTypes[1] = utils.Uint128
@@ -139,7 +139,7 @@ var _ = Describe("Storage decoder", func() {
 				PackedTypes: packedTypes,
 			}
 
-			result, err := utils.Decode(row, metadata)
+			result, err := utils.Decode(diff, metadata)
 			decodedValues := result.(map[int]string)
 
 			Expect(err).NotTo(HaveOccurred())
@@ -151,7 +151,7 @@ var _ = Describe("Storage decoder", func() {
 			//TODO: replace with real data when available
 			addressHex := "0000000000000000000000000000000000012345"
 			packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			row := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Address
 			packedTypes[1] = utils.Uint48

libraries/shared/storage/utils/diff.go

@@ -0,0 +1,73 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"strconv"
+)
+
+const ExpectedRowLength = 5
+
+type StorageDiff struct {
+	Id            int
+	HashedAddress common.Hash `db:"contract"`
+	BlockHash     common.Hash `db:"block_hash"`
+	BlockHeight   int         `db:"block_height"`
+	StorageKey    common.Hash `db:"storage_key"`
+	StorageValue  common.Hash `db:"storage_value"`
+}
+
+func FromParityCsvRow(csvRow []string) (StorageDiff, error) {
+	if len(csvRow) != ExpectedRowLength {
+		return StorageDiff{}, ErrRowMalformed{Length: len(csvRow)}
+	}
+	height, err := strconv.Atoi(csvRow[2])
+	if err != nil {
+		return StorageDiff{}, err
+	}
+	return StorageDiff{
+		HashedAddress: HexToKeccak256Hash(csvRow[0]),
+		BlockHash:     common.HexToHash(csvRow[1]),
+		BlockHeight:   height,
+		StorageKey:    common.HexToHash(csvRow[3]),
+		StorageValue:  common.HexToHash(csvRow[4]),
+	}, nil
+}
+
+func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiff, error) {
+	var decodedValue []byte
+	err := rlp.DecodeBytes(storage.Value, &decodedValue)
+	if err != nil {
+		return StorageDiff{}, err
+	}
+
+	return StorageDiff{
+		HashedAddress: common.BytesToHash(account.Key),
+		BlockHash:     stateDiff.BlockHash,
+		BlockHeight:   int(stateDiff.BlockNumber.Int64()),
+		StorageKey:    common.BytesToHash(storage.Key),
+		StorageValue:  common.BytesToHash(decodedValue),
+	}, nil
+}
+
+func HexToKeccak256Hash(hex string) common.Hash {
+	return crypto.Keccak256Hash(common.FromHex(hex))
+}

libraries/shared/storage/utils/diff_test.go

@@ -0,0 +1,121 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"math/big"
+	"math/rand"
+)
+
+var _ = Describe("Storage row parsing", func() {
+	Describe("FromParityCsvRow", func() {
+		It("converts an array of strings to a row struct", func() {
+			contract := "0x123"
+			blockHash := "0x456"
+			blockHeight := "789"
+			storageKey := "0x987"
+			storageValue := "0x654"
+			data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
+
+			result, err := utils.FromParityCsvRow(data)
+
+			Expect(err).NotTo(HaveOccurred())
+			expectedKeccakOfContractAddress := utils.HexToKeccak256Hash(contract)
+			Expect(result.HashedAddress).To(Equal(expectedKeccakOfContractAddress))
+			Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
+			Expect(result.BlockHeight).To(Equal(789))
+			Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
+			Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
+		})
+
+		It("returns an error if row is missing data", func() {
+			_, err := utils.FromParityCsvRow([]string{"0x123"})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
+		})
+
+		It("returns error if block height malformed", func() {
+			_, err := utils.FromParityCsvRow([]string{"", "", "", "", ""})
+
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("FromGethStateDiff", func() {
+		var (
+			accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
+			stateDiff   = &statediff.StateDiff{
+				BlockNumber: big.NewInt(rand.Int63()),
+				BlockHash:   fakes.FakeHash,
+			}
+		)
+
+		It("adds relevant fields to diff", func() {
+			storageValueBytes := []byte{3}
+			storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
+			Expect(encodeErr).NotTo(HaveOccurred())
+
+			storageDiff := statediff.StorageDiff{
+				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
+				Value: storageValueRlp,
+			}
+
+			result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).NotTo(HaveOccurred())
+
+			expectedAddress := common.BytesToHash(accountDiff.Key)
+			Expect(result.HashedAddress).To(Equal(expectedAddress))
+			Expect(result.BlockHash).To(Equal(fakes.FakeHash))
+			expectedBlockHeight := int(stateDiff.BlockNumber.Int64())
+			Expect(result.BlockHeight).To(Equal(expectedBlockHeight))
+			expectedStorageKey := common.BytesToHash(storageDiff.Key)
+			Expect(result.StorageKey).To(Equal(expectedStorageKey))
+			expectedStorageValue := common.BytesToHash(storageValueBytes)
+			Expect(result.StorageValue).To(Equal(expectedStorageValue))
+		})
+
+		It("handles decoding large storage values from their RLP", func() {
+			storageValueBytes := []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6}
+			storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
+			Expect(encodeErr).NotTo(HaveOccurred())
+
+			storageDiff := statediff.StorageDiff{
+				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
+				Value: storageValueRlp,
+			}
+
+			result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes)))
+		})
+
+		It("returns an err if decoding the storage value Rlp fails", func() {
+			_, err := utils.FromGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("rlp: input contains more than one value"))
+		})
+	})
+})

libraries/shared/storage/utils/keys_loader.go

@@ -14,23 +14,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package storage
+package utils
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"math/big"
 )
 
-type Mappings interface {
-	Lookup(key common.Hash) (utils.StorageValueMetadata, error)
-	SetDB(db *postgres.DB)
-}
-
 const (
 	IndexZero   = "0000000000000000000000000000000000000000000000000000000000000000"
 	IndexOne    = "0000000000000000000000000000000000000000000000000000000000000001"
@@ -46,19 +37,17 @@ const (
 	IndexEleven = "000000000000000000000000000000000000000000000000000000000000000b"
 )
 
-func GetMapping(indexOnContract, key string) common.Hash {
+func GetStorageKeyForMapping(indexOnContract, key string) common.Hash {
 	keyBytes := common.FromHex(key + indexOnContract)
-	encoded := crypto.Keccak256(keyBytes)
-	return common.BytesToHash(encoded)
+	return crypto.Keccak256Hash(keyBytes)
 }
 
-func GetNestedMapping(indexOnContract, primaryKey, secondaryKey string) common.Hash {
+func GetStorageKeyForNestedMapping(indexOnContract, primaryKey, secondaryKey string) common.Hash {
 	primaryMappingIndex := crypto.Keccak256(common.FromHex(primaryKey + indexOnContract))
-	secondaryMappingIndex := crypto.Keccak256(common.FromHex(secondaryKey), primaryMappingIndex)
-	return common.BytesToHash(secondaryMappingIndex)
+	return crypto.Keccak256Hash(common.FromHex(secondaryKey), primaryMappingIndex)
 }
 
-func GetIncrementedKey(original common.Hash, incrementBy int64) common.Hash {
+func GetIncrementedStorageKey(original common.Hash, incrementBy int64) common.Hash {
 	originalMappingAsInt := original.Big()
 	incremented := big.NewInt(0).Add(originalMappingAsInt, big.NewInt(incrementBy))
 	return common.BytesToHash(incremented.Bytes())

libraries/shared/storage/utils/keys_loader_test.go

@@ -1,56 +1,72 @@
-package storage_test
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 )
 
-var _ = Describe("Mappings", func() {
-	Describe("GetMapping", func() {
+var _ = Describe("Storage keys loader utils", func() {
+	Describe("GetStorageKeyForMapping", func() {
 		It("returns the storage key for a mapping when passed the mapping's index on the contract and the desired value's key", func() {
 			// ex. solidity:
 			//    	mapping (bytes32 => uint) public amounts
 			// to access amounts, pass in the index of the mapping on the contract + the bytes32 key for the uint val being looked up
-			indexOfMappingOnContract := storage.IndexZero
+			indexOfMappingOnContract := utils.IndexZero
 			keyForDesiredValueInMapping := "1234567890abcdef"
 
-			storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+			storageKey := utils.GetStorageKeyForMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
 
 			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
 			Expect(storageKey).To(Equal(expectedStorageKey))
 		})
 
 		It("returns same result if value includes hex prefix", func() {
-			indexOfMappingOnContract := storage.IndexZero
+			indexOfMappingOnContract := utils.IndexZero
 			keyForDesiredValueInMapping := "0x1234567890abcdef"
 
-			storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+			storageKey := utils.GetStorageKeyForMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
 
 			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
 			Expect(storageKey).To(Equal(expectedStorageKey))
 		})
 	})
 
-	Describe("GetNestedMapping", func() {
+	Describe("GetStorageKeyForNestedMapping", func() {
 		It("returns the storage key for a nested mapping when passed the mapping's index on the contract and the desired value's keys", func() {
 			// ex. solidity:
 			//    	mapping (bytes32 => uint) public amounts
 			//    	mapping (address => mapping (uint => bytes32)) public addressNames
 			// to access addressNames, pass in the index of the mapping on the contract + the address and uint keys for the bytes32 val being looked up
-			indexOfMappingOnContract := storage.IndexOne
+			indexOfMappingOnContract := utils.IndexOne
 			keyForOuterMapping := "1234567890abcdef"
 			keyForInnerMapping := "123"
 
-			storageKey := storage.GetNestedMapping(indexOfMappingOnContract, keyForOuterMapping, keyForInnerMapping)
+			storageKey := utils.GetStorageKeyForNestedMapping(indexOfMappingOnContract, keyForOuterMapping, keyForInnerMapping)
 
 			expectedStorageKey := common.HexToHash("0x82113529f6cd61061d1a6f0de53f2bdd067a1addd3d2b46be50a99abfcdb1661")
 			Expect(storageKey).To(Equal(expectedStorageKey))
 		})
 	})
 
-	Describe("GetIncrementedKey", func() {
+	Describe("GetIncrementedStorageKey", func() {
 		It("returns the storage key for later values sharing an index on the contract with other earlier values", func() {
 			// ex. solidity:
 			//    	mapping (bytes32 => uint) public amounts
@@ -62,11 +78,11 @@ var _ = Describe("Mappings", func() {
 			//    	mapping (bytes32 => Data) public itemData;
 			// to access quality from itemData, pass in the storage key for the zero-indexed value (quantity) + the number of increments required.
 			// (For "quality", we must increment the storage key for the corresponding "quantity" by 1).
-			indexOfMappingOnContract := storage.IndexTwo
+			indexOfMappingOnContract := utils.IndexTwo
 			keyForDesiredValueInMapping := "1234567890abcdef"
-			storageKeyForFirstPropertyOnStruct := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+			storageKeyForFirstPropertyOnStruct := utils.GetStorageKeyForMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
 
-			storageKey := storage.GetIncrementedKey(storageKeyForFirstPropertyOnStruct, 1)
+			storageKey := utils.GetIncrementedStorageKey(storageKeyForFirstPropertyOnStruct, 1)
 
 			expectedStorageKey := common.HexToHash("0x69b38749f0a8ed5d505c8474f7fb62c7828aad8a7627f1c67e07af1d2368cad4")
 			Expect(storageKey).To(Equal(expectedStorageKey))

libraries/shared/storage/utils/keys_lookup.go

@@ -17,35 +17,21 @@
 package utils
 
 import (
-	"strconv"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 )
 
-const ExpectedRowLength = 5
-
-type StorageDiffRow struct {
-	Id           int
-	Contract     common.Address
-	BlockHash    common.Hash `db:"block_hash"`
-	BlockHeight  int         `db:"block_height"`
-	StorageKey   common.Hash `db:"storage_key"`
-	StorageValue common.Hash `db:"storage_value"`
+func AddHashedKeys(currentMappings map[common.Hash]StorageValueMetadata) map[common.Hash]StorageValueMetadata {
+	copyOfCurrentMappings := make(map[common.Hash]StorageValueMetadata)
+	for k, v := range currentMappings {
+		copyOfCurrentMappings[k] = v
+	}
+	for k, v := range copyOfCurrentMappings {
+		currentMappings[hashKey(k)] = v
+	}
+	return currentMappings
 }
 
-func FromStrings(csvRow []string) (StorageDiffRow, error) {
-	if len(csvRow) != ExpectedRowLength {
-		return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)}
-	}
-	height, err := strconv.Atoi(csvRow[2])
-	if err != nil {
-		return StorageDiffRow{}, err
-	}
-	return StorageDiffRow{
-		Contract:     common.HexToAddress(csvRow[0]),
-		BlockHash:    common.HexToHash(csvRow[1]),
-		BlockHeight:  height,
-		StorageKey:   common.HexToHash(csvRow[3]),
-		StorageValue: common.HexToHash(csvRow[4]),
-	}, nil
+func hashKey(key common.Hash) common.Hash {
+	return crypto.Keccak256Hash(key.Bytes())
 }

libraries/shared/storage/utils/keys_lookup_test.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
+
+var _ = Describe("Storage keys lookup utils", func() {
+	Describe("AddHashedKeys", func() {
+		It("returns a copy of the map with an additional slot for the hashed version of every key", func() {
+			fakeMap := map[common.Hash]utils.StorageValueMetadata{}
+			fakeStorageKey := common.HexToHash("72c72de6b203d67cb6cd54fc93300109fcc6fd6eac88e390271a3d548794d800")
+			var fakeMappingKey utils.Key = "fakeKey"
+			fakeMetadata := utils.StorageValueMetadata{
+				Name: "fakeName",
+				Keys: map[utils.Key]string{fakeMappingKey: "fakeValue"},
+				Type: utils.Uint48,
+			}
+			fakeMap[fakeStorageKey] = fakeMetadata
+
+			result := utils.AddHashedKeys(fakeMap)
+
+			Expect(len(result)).To(Equal(2))
+			expectedHashedStorageKey := common.HexToHash("2165edb4e1c37b99b60fa510d84f939dd35d5cd1d1c8f299d6456ea09df65a76")
+			Expect(fakeMap[fakeStorageKey]).To(Equal(fakeMetadata))
+			Expect(fakeMap[expectedHashedStorageKey]).To(Equal(fakeMetadata))
+		})
+	})
+})

libraries/shared/storage/utils/row_test.go

@@ -1,58 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package utils_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-)
-
-var _ = Describe("Storage row parsing", func() {
-	It("converts an array of strings to a row struct", func() {
-		contract := "0x123"
-		blockHash := "0x456"
-		blockHeight := "789"
-		storageKey := "0x987"
-		storageValue := "0x654"
-		data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
-
-		result, err := utils.FromStrings(data)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
-		Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
-		Expect(result.BlockHeight).To(Equal(789))
-		Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
-		Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
-	})
-
-	It("returns an error if row is missing data", func() {
-		_, err := utils.FromStrings([]string{"0x123"})
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
-	})
-
-	It("returns error if block height malformed", func() {
-		_, err := utils.FromStrings([]string{"", "", "", "", ""})
-
-		Expect(err).To(HaveOccurred())
-	})
-})

libraries/shared/streamer/statediff_streamer.go

@@ -0,0 +1,41 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package streamer
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type Streamer interface {
+	Stream(chan statediff.Payload) (*rpc.ClientSubscription, error)
+}
+
+type StateDiffStreamer struct {
+	client core.RpcClient
+}
+
+func (streamer *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	logrus.Info("streaming diffs from geth")
+	return streamer.client.Subscribe("statediff", payloadChan, "stream")
+}
+
+func NewStateDiffStreamer(client core.RpcClient) StateDiffStreamer {
+	return StateDiffStreamer{
+		client: client,
+	}
+}

libraries/shared/streamer/statediff_streamer_test.go

@@ -0,0 +1,35 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package streamer_test
+
+import (
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("StateDiff Streamer", func() {
+	It("subscribes to the geth statediff service", func() {
+		client := &fakes.MockRpcClient{}
+		streamer := streamer.NewStateDiffStreamer(client)
+		payloadChan := make(chan statediff.Payload)
+		_, err := streamer.Stream(payloadChan)
+		Expect(err).NotTo(HaveOccurred())
+
+		client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"})
+	})
+})

libraries/shared/streamer/streamer_suite_test.go

@@ -0,0 +1,19 @@
+package streamer_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestStreamer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Streamer Suite")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

libraries/shared/test_data/generic.go

@@ -20,14 +20,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"math/rand"
 	"time"
 )
 
-type GenericModel struct{}
-type GenericEntity struct{}
-
 var startingBlockNumber = rand.Int63()
 var topic0 = "0x" + randomString(64)
 
@@ -44,6 +42,8 @@ var GenericTestLog = func() types.Log {
 	}
 }
 
+var GenericModel = event.InsertionModel{}
+
 var GenericTestConfig = transformer.EventTransformerConfig{
 	TransformerName:     "generic-test-transformer",
 	ContractAddresses:   []string{fakeAddress().Hex()},

libraries/shared/test_data/statediff.go

@@ -0,0 +1,137 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test_data
+
+import (
+	"errors"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"math/big"
+	"math/rand"
+)
+
+var (
+	BlockNumber             = big.NewInt(rand.Int63())
+	BlockHash               = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
+	CodeHash                = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	NewNonceValue           = rand.Uint64()
+	NewBalanceValue         = rand.Int63()
+	ContractRoot            = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+	StoragePath             = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+	StorageKey              = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+	SmallStorageValue       = common.Hex2Bytes("03")
+	SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue)
+	storageWithSmallValue   = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: SmallStorageValueRlp,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+	}}
+	LargeStorageValue            = common.Hex2Bytes("00191b53778c567b14b50ba0000")
+	LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue)
+	storageWithLargeValue        = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: LargeStorageValueRlp,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+	}}
+	EmptyStorage        = make([]statediff.StorageDiff, 0)
+	StorageWithBadValue = statediff.StorageDiff{
+		Key:   StorageKey,
+		Value: []byte{0, 1, 2},
+		// this storage value will fail to be decoded as an RLP with the following error message:
+		// "input contains more than one value"
+	}
+	contractAddress        = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	ContractLeafKey        = crypto.Keccak256Hash(contractAddress[:])
+	anotherContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherContractLeafKey = crypto.Keccak256Hash(anotherContractAddress[:])
+
+	testAccount = state.Account{
+		Nonce:    NewNonceValue,
+		Balance:  big.NewInt(NewBalanceValue),
+		Root:     ContractRoot,
+		CodeHash: CodeHash,
+	}
+	valueBytes, _       = rlp.EncodeToBytes(testAccount)
+	CreatedAccountDiffs = []statediff.AccountDiff{
+		{
+			Key:     ContractLeafKey.Bytes(),
+			Value:   valueBytes,
+			Storage: storageWithSmallValue,
+		},
+	}
+
+	UpdatedAccountDiffs = []statediff.AccountDiff{{
+		Key:     AnotherContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storageWithLargeValue,
+	}}
+
+	DeletedAccountDiffs = []statediff.AccountDiff{{
+		Key:     AnotherContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storageWithSmallValue,
+	}}
+
+	MockStateDiff = statediff.StateDiff{
+		BlockNumber:     BlockNumber,
+		BlockHash:       common.HexToHash(BlockHash),
+		CreatedAccounts: CreatedAccountDiffs,
+		DeletedAccounts: DeletedAccountDiffs,
+		UpdatedAccounts: UpdatedAccountDiffs,
+	}
+	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
+
+	mockTransaction1 = types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil)
+	mockTransaction2 = types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil)
+	MockTransactions = types.Transactions{mockTransaction1, mockTransaction2}
+
+	mockReceipt1 = types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt2 = types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	MockReceipts = types.Receipts{mockReceipt1, mockReceipt2}
+
+	MockHeader = types.Header{
+		Time:        0,
+		Number:      BlockNumber,
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+	}
+	MockBlock       = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock)
+
+	MockStatediffPayload = statediff.Payload{
+		BlockRlp:     MockBlockRlp,
+		StateDiffRlp: MockStateDiffBytes,
+		Err:          nil,
+	}
+
+	EmptyStatediffPayload = statediff.Payload{
+		BlockRlp:     []byte{},
+		StateDiffRlp: []byte{},
+		Err:          nil,
+	}
+
+	ErrStatediffPayload = statediff.Payload{
+		BlockRlp:     []byte{},
+		StateDiffRlp: []byte{},
+		Err:          errors.New("mock error"),
+	}
+)

libraries/shared/test_data/test_helpers.go

@@ -0,0 +1,37 @@
+package test_data
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"math/rand"
+)
+
+// Create a header sync log to reference in an event, returning inserted header sync log
+func CreateTestLog(headerID int64, db *postgres.DB) core.HeaderSyncLog {
+	log := types.Log{
+		Address:     common.Address{},
+		Topics:      nil,
+		Data:        nil,
+		BlockNumber: 0,
+		TxHash:      common.Hash{},
+		TxIndex:     uint(rand.Int31()),
+		BlockHash:   common.Hash{},
+		Index:       0,
+		Removed:     false,
+	}
+	headerSyncLogRepository := repositories.NewHeaderSyncLogRepository(db)
+	insertLogsErr := headerSyncLogRepository.CreateHeaderSyncLogs(headerID, []types.Log{log})
+	Expect(insertLogsErr).NotTo(HaveOccurred())
+	headerSyncLogs, getLogsErr := headerSyncLogRepository.GetUntransformedHeaderSyncLogs()
+	Expect(getLogsErr).NotTo(HaveOccurred())
+	for _, headerSyncLog := range headerSyncLogs {
+		if headerSyncLog.Log.TxIndex == log.TxIndex {
+			return headerSyncLog
+		}
+	}
+	panic("couldn't find inserted test log")
+}

libraries/shared/transformer/storage_transformer.go

@@ -18,14 +18,13 @@ package transformer
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type StorageTransformer interface {
-	Execute(row utils.StorageDiffRow) error
-	ContractAddress() common.Address
+	Execute(diff utils.StorageDiff) error
+	KeccakContractAddress() common.Hash
 }
 
 type StorageTransformerInitializer func(db *postgres.DB) StorageTransformer

libraries/shared/watcher/storage_watcher.go

@@ -18,69 +18,76 @@ package watcher
 
 import (
 	"fmt"
-	"reflect"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"time"
 )
 
+type IStorageWatcher interface {
+	AddTransformers(initializers []transformer.StorageTransformerInitializer)
+	Execute(diffsChan chan utils.StorageDiff, errsChan chan error, queueRecheckInterval time.Duration)
+}
+
 type StorageWatcher struct {
-	db             *postgres.DB
-	StorageFetcher fetcher.IStorageFetcher
-	Queue          storage.IStorageQueue
-	Transformers   map[common.Address]transformer.StorageTransformer
+	db                        *postgres.DB
+	StorageFetcher            fetcher.IStorageFetcher
+	Queue                     storage.IStorageQueue
+	KeccakAddressTransformers map[common.Hash]transformer.StorageTransformer // keccak hash of an address => transformer
 }
 
 func NewStorageWatcher(fetcher fetcher.IStorageFetcher, db *postgres.DB) StorageWatcher {
-	transformers := make(map[common.Address]transformer.StorageTransformer)
 	queue := storage.NewStorageQueue(db)
+	transformers := make(map[common.Hash]transformer.StorageTransformer)
 	return StorageWatcher{
-		db:             db,
-		StorageFetcher: fetcher,
-		Queue:          queue,
-		Transformers:   transformers,
+		db:                        db,
+		StorageFetcher:            fetcher,
+		Queue:                     queue,
+		KeccakAddressTransformers: transformers,
 	}
 }
 
 func (storageWatcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
 		storageTransformer := initializer(storageWatcher.db)
-		storageWatcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
+		storageWatcher.KeccakAddressTransformers[storageTransformer.KeccakContractAddress()] = storageTransformer
 	}
 }
 
-func (storageWatcher StorageWatcher) Execute(rows chan utils.StorageDiffRow, errs chan error, queueRecheckInterval time.Duration) {
+func (storageWatcher StorageWatcher) Execute(diffsChan chan utils.StorageDiff, errsChan chan error, queueRecheckInterval time.Duration) {
 	ticker := time.NewTicker(queueRecheckInterval)
-	go storageWatcher.StorageFetcher.FetchStorageDiffs(rows, errs)
+	go storageWatcher.StorageFetcher.FetchStorageDiffs(diffsChan, errsChan)
 	for {
 		select {
-		case fetchErr := <-errs:
+		case fetchErr := <-errsChan:
 			logrus.Warn(fmt.Sprintf("error fetching storage diffs: %s", fetchErr))
-		case row := <-rows:
-			storageWatcher.processRow(row)
+		case diff := <-diffsChan:
+			storageWatcher.processRow(diff)
 		case <-ticker.C:
 			storageWatcher.processQueue()
 		}
 	}
 }
 
-func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
-	storageTransformer, ok := storageWatcher.Transformers[row.Contract]
+func (storageWatcher StorageWatcher) getTransformer(diff utils.StorageDiff) (transformer.StorageTransformer, bool) {
+	storageTransformer, ok := storageWatcher.KeccakAddressTransformers[diff.HashedAddress]
+	return storageTransformer, ok
+}
+
+func (storageWatcher StorageWatcher) processRow(diff utils.StorageDiff) {
+	storageTransformer, ok := storageWatcher.getTransformer(diff)
 	if !ok {
-		// ignore rows from unwatched contracts
+		logrus.Debug("ignoring a diff from an unwatched contract")
 		return
 	}
-	executeErr := storageTransformer.Execute(row)
+	executeErr := storageTransformer.Execute(diff)
 	if executeErr != nil {
 		logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
-		queueErr := storageWatcher.Queue.Add(row)
+		queueErr := storageWatcher.Queue.Add(diff)
 		if queueErr != nil {
 			logrus.Warn(fmt.Sprintf("error queueing storage diff: %s", queueErr))
 		}
@@ -88,20 +95,20 @@ func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
 }
 
 func (storageWatcher StorageWatcher) processQueue() {
-	rows, fetchErr := storageWatcher.Queue.GetAll()
+	diffs, fetchErr := storageWatcher.Queue.GetAll()
 	if fetchErr != nil {
 		logrus.Warn(fmt.Sprintf("error getting queued storage: %s", fetchErr))
 	}
-	for _, row := range rows {
-		storageTransformer, ok := storageWatcher.Transformers[row.Contract]
+	for _, diff := range diffs {
+		storageTransformer, ok := storageWatcher.getTransformer(diff)
 		if !ok {
-			// delete row from queue if address no longer watched
-			storageWatcher.deleteRow(row.Id)
+			// delete diff from queue if address no longer watched
+			storageWatcher.deleteRow(diff.Id)
 			continue
 		}
-		executeErr := storageTransformer.Execute(row)
+		executeErr := storageTransformer.Execute(diff)
 		if executeErr == nil {
-			storageWatcher.deleteRow(row.Id)
+			storageWatcher.deleteRow(diff.Id)
 		}
 	}
 }
@@ -109,10 +116,6 @@ func (storageWatcher StorageWatcher) processQueue() {
 func (storageWatcher StorageWatcher) deleteRow(id int) {
 	deleteErr := storageWatcher.Queue.Delete(id)
 	if deleteErr != nil {
-		logrus.Warn(fmt.Sprintf("error deleting persisted row from queue: %s", deleteErr))
+		logrus.Warn(fmt.Sprintf("error deleting persisted diff from queue: %s", deleteErr))
 	}
 }
-
-func isKeyNotFound(executeErr error) bool {
-	return reflect.TypeOf(executeErr) == reflect.TypeOf(utils.ErrStorageKeyNotFound{})
-}

libraries/shared/watcher/storage_watcher_test.go

@@ -17,59 +17,60 @@
 package watcher_test
 
 import (
-	"io/ioutil"
-	"os"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/test_config"
+	"io/ioutil"
+	"os"
+	"time"
 )
 
 var _ = Describe("Storage Watcher", func() {
-	It("adds transformers", func() {
-		fakeAddress := common.HexToAddress("0x12345")
-		fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
-		w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))
+	Describe("AddTransformer", func() {
+		It("adds transformers", func() {
+			fakeHashedAddress := utils.HexToKeccak256Hash("0x12345")
+			fakeTransformer := &mocks.MockStorageTransformer{KeccakOfAddress: fakeHashedAddress}
+			w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))
 
-		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
 
-		Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
+			Expect(w.KeccakAddressTransformers[fakeHashedAddress]).To(Equal(fakeTransformer))
+		})
 	})
 
-	Describe("executing watcher", func() {
+	Describe("Execute", func() {
 		var (
 			errs            chan error
 			mockFetcher     *mocks.MockStorageFetcher
 			mockQueue       *mocks.MockStorageQueue
 			mockTransformer *mocks.MockStorageTransformer
-			row             utils.StorageDiffRow
-			rows            chan utils.StorageDiffRow
+			csvDiff         utils.StorageDiff
+			diffs           chan utils.StorageDiff
 			storageWatcher  watcher.StorageWatcher
+			hashedAddress   common.Hash
 		)
 
 		BeforeEach(func() {
 			errs = make(chan error)
-			rows = make(chan utils.StorageDiffRow)
-			address := common.HexToAddress("0x0123456789abcdef")
+			diffs = make(chan utils.StorageDiff)
+			hashedAddress = utils.HexToKeccak256Hash("0x0123456789abcdef")
 			mockFetcher = mocks.NewMockStorageFetcher()
 			mockQueue = &mocks.MockStorageQueue{}
-			mockTransformer = &mocks.MockStorageTransformer{Address: address}
-			row = utils.StorageDiffRow{
-				Id:           1337,
-				Contract:     address,
-				BlockHash:    common.HexToHash("0xfedcba9876543210"),
-				BlockHeight:  0,
-				StorageKey:   common.HexToHash("0xabcdef1234567890"),
-				StorageValue: common.HexToHash("0x9876543210abcdef"),
+			mockTransformer = &mocks.MockStorageTransformer{KeccakOfAddress: hashedAddress}
+			csvDiff = utils.StorageDiff{
+				Id:            1337,
+				HashedAddress: hashedAddress,
+				BlockHash:     common.HexToHash("0xfedcba9876543210"),
+				BlockHeight:   0,
+				StorageKey:    common.HexToHash("0xabcdef1234567890"),
+				StorageValue:  common.HexToHash("0x9876543210abcdef"),
 			}
 		})
 
@@ -83,7 +84,7 @@ var _ = Describe("Storage Watcher", func() {
 			defer os.Remove(tempFile.Name())
 			logrus.SetOutput(tempFile)
 
-			go storageWatcher.Execute(rows, errs, time.Hour)
+			go storageWatcher.Execute(diffs, errs, time.Hour)
 
 			Eventually(func() (string, error) {
 				logContent, err := ioutil.ReadFile(tempFile.Name())
@@ -92,39 +93,39 @@ var _ = Describe("Storage Watcher", func() {
 			close(done)
 		})
 
-		Describe("transforming new storage diffs", func() {
+		Describe("transforming new storage diffs from csv", func() {
 			BeforeEach(func() {
-				mockFetcher.RowsToReturn = []utils.StorageDiffRow{row}
+				mockFetcher.DiffsToReturn = []utils.StorageDiff{csvDiff}
 				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
 				storageWatcher.Queue = mockQueue
 				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
 			})
 
-			It("executes transformer for recognized storage row", func(done Done) {
-				go storageWatcher.Execute(rows, errs, time.Hour)
+			It("executes transformer for recognized storage diff", func(done Done) {
+				go storageWatcher.Execute(diffs, errs, time.Hour)
 
-				Eventually(func() utils.StorageDiffRow {
-					return mockTransformer.PassedRow
-				}).Should(Equal(row))
+				Eventually(func() utils.StorageDiff {
+					return mockTransformer.PassedDiff
+				}).Should(Equal(csvDiff))
 				close(done)
 			})
 
-			It("queues row for later processing if transformer execution fails", func(done Done) {
+			It("queues diff for later processing if transformer execution fails", func(done Done) {
 				mockTransformer.ExecuteErr = fakes.FakeError
 
-				go storageWatcher.Execute(rows, errs, time.Hour)
+				go storageWatcher.Execute(diffs, errs, time.Hour)
 
 				Expect(<-errs).To(BeNil())
 				Eventually(func() bool {
 					return mockQueue.AddCalled
 				}).Should(BeTrue())
-				Eventually(func() utils.StorageDiffRow {
-					return mockQueue.AddPassedRow
-				}).Should(Equal(row))
+				Eventually(func() utils.StorageDiff {
+					return mockQueue.AddPassedDiff
+				}).Should(Equal(csvDiff))
 				close(done)
 			})
 
-			It("logs error if queueing row fails", func(done Done) {
+			It("logs error if queueing diff fails", func(done Done) {
 				mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
 				mockQueue.AddError = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")
@@ -132,7 +133,7 @@ var _ = Describe("Storage Watcher", func() {
 				defer os.Remove(tempFile.Name())
 				logrus.SetOutput(tempFile)
 
-				go storageWatcher.Execute(rows, errs, time.Hour)
+				go storageWatcher.Execute(diffs, errs, time.Hour)
 
 				Eventually(func() bool {
 					return mockQueue.AddCalled
@@ -147,20 +148,38 @@ var _ = Describe("Storage Watcher", func() {
 
 		Describe("transforming queued storage diffs", func() {
 			BeforeEach(func() {
-				mockQueue.RowsToReturn = []utils.StorageDiffRow{row}
+				mockQueue.DiffsToReturn = []utils.StorageDiff{csvDiff}
 				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
 				storageWatcher.Queue = mockQueue
 				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
 			})
 
-			It("logs error if getting queued storage fails", func(done Done) {
-				mockQueue.GetAllErr = fakes.FakeError
+			It("executes transformer for storage diff", func(done Done) {
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
+
+				Eventually(func() utils.StorageDiff {
+					return mockTransformer.PassedDiff
+				}).Should(Equal(csvDiff))
+				close(done)
+			})
+
+			It("deletes diff from queue if transformer execution successful", func(done Done) {
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(csvDiff.Id))
+				close(done)
+			})
+
+			It("logs error if deleting persisted diff fails", func(done Done) {
+				mockQueue.DeleteErr = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")
 				Expect(fileErr).NotTo(HaveOccurred())
 				defer os.Remove(tempFile.Name())
 				logrus.SetOutput(tempFile)
 
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
 
 				Eventually(func() (string, error) {
 					logContent, err := ioutil.ReadFile(tempFile.Name())
@@ -169,68 +188,34 @@ var _ = Describe("Storage Watcher", func() {
 				close(done)
 			})
 
-			It("executes transformer for storage row", func(done Done) {
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+			It("deletes obsolete diff from queue if contract not recognized", func(done Done) {
+				obsoleteDiff := utils.StorageDiff{
+					Id:            csvDiff.Id + 1,
+					HashedAddress: utils.HexToKeccak256Hash("0xfedcba9876543210"),
+				}
+				mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
 
-				Eventually(func() utils.StorageDiffRow {
-					return mockTransformer.PassedRow
-				}).Should(Equal(row))
-				close(done)
-			})
-
-			It("deletes row from queue if transformer execution successful", func(done Done) {
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
 
 				Eventually(func() int {
 					return mockQueue.DeletePassedId
-				}).Should(Equal(row.Id))
+				}).Should(Equal(obsoleteDiff.Id))
 				close(done)
 			})
 
-			It("logs error if deleting persisted row fails", func(done Done) {
+			It("logs error if deleting obsolete diff fails", func(done Done) {
+				obsoleteDiff := utils.StorageDiff{
+					Id:            csvDiff.Id + 1,
+					HashedAddress: utils.HexToKeccak256Hash("0xfedcba9876543210"),
+				}
+				mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
 				mockQueue.DeleteErr = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")
 				Expect(fileErr).NotTo(HaveOccurred())
 				defer os.Remove(tempFile.Name())
 				logrus.SetOutput(tempFile)
 
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
-
-				Eventually(func() (string, error) {
-					logContent, err := ioutil.ReadFile(tempFile.Name())
-					return string(logContent), err
-				}).Should(ContainSubstring(fakes.FakeError.Error()))
-				close(done)
-			})
-
-			It("deletes obsolete row from queue if contract not recognized", func(done Done) {
-				obsoleteRow := utils.StorageDiffRow{
-					Id:       row.Id + 1,
-					Contract: common.HexToAddress("0xfedcba9876543210"),
-				}
-				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
-
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
-
-				Eventually(func() int {
-					return mockQueue.DeletePassedId
-				}).Should(Equal(obsoleteRow.Id))
-				close(done)
-			})
-
-			It("logs error if deleting obsolete row fails", func(done Done) {
-				obsoleteRow := utils.StorageDiffRow{
-					Id:       row.Id + 1,
-					Contract: common.HexToAddress("0xfedcba9876543210"),
-				}
-				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
-				mockQueue.DeleteErr = fakes.FakeError
-				tempFile, fileErr := ioutil.TempFile("", "log")
-				Expect(fileErr).NotTo(HaveOccurred())
-				defer os.Remove(tempFile.Name())
-				logrus.SetOutput(tempFile)
-
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
 
 				Eventually(func() (string, error) {
 					logContent, err := ioutil.ReadFile(tempFile.Name())
@@ -239,6 +224,5 @@ var _ = Describe("Storage Watcher", func() {
 				close(done)
 			})
 		})
-
 	})
 })

pkg/config/contract.go

@@ -19,7 +19,7 @@ package config
 import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"strings"
 )
 
@@ -98,7 +98,7 @@ func (contractConfig *ContractConfig) PrepConfig() {
 			}
 		}
 		if abi != "" {
-			if _, abiErr := geth.ParseAbi(abi); abiErr != nil {
+			if _, abiErr := eth.ParseAbi(abi); abiErr != nil {
 				log.Fatal(addr, "transformer `abi` not valid JSON")
 			}
 		}

pkg/contract_watcher/full/converter/converter.go

@@ -17,7 +17,6 @@
 package converter
 
 import (
-	"errors"
 	"fmt"
 	"math/big"
 	"strconv"
@@ -32,22 +31,24 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-// Converter is used to convert watched event logs to
+// ConverterInterface is used to convert watched event logs to
 // custom logs containing event input name => value maps
 type ConverterInterface interface {
 	Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error)
 	Update(info *contract.Contract)
 }
 
+// Converter is the underlying struct for the ConverterInterface
 type Converter struct {
 	ContractInfo *contract.Contract
 }
 
+// Update configures the converter for a specific contract
 func (c *Converter) Update(info *contract.Contract) {
 	c.ContractInfo = info
 }
 
-// Convert the given watched event log into a types.Log for the given event
+// Convert converts the given watched event log into a types.Log for the given event
 func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
 	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	values := make(map[string]interface{})
@@ -88,14 +89,14 @@ func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
 			b := input.(byte)
 			strValues[fieldName] = string(b)
 		default:
-			return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
+			return nil, fmt.Errorf("error: unhandled abi type %T", input)
 		}
 	}
 
 	// Only hold onto logs that pass our address filter, if any
 	if c.ContractInfo.PassesEventFilter(strValues) {
 		eventLog := &types.Log{
-			Id:     watchedEvent.LogID,
+			ID:     watchedEvent.LogID,
 			Values: strValues,
 			Block:  watchedEvent.BlockNumber,
 			Tx:     watchedEvent.TxHash,

pkg/contract_watcher/full/retriever/block_retriever.go

@@ -18,11 +18,12 @@ package retriever
 
 import (
 	"database/sql"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
-// Block retriever is used to retrieve the first block for a given contract and the most recent block
+// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
 // It requires a vDB synced database with blocks, transactions, receipts, and logs
 type BlockRetriever interface {
 	RetrieveFirstBlock(contractAddr string) (int64, error)
@@ -33,13 +34,15 @@ type blockRetriever struct {
 	db *postgres.DB
 }
 
-func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
+// NewBlockRetriever returns a new BlockRetriever
+func NewBlockRetriever(db *postgres.DB) BlockRetriever {
 	return &blockRetriever{
 		db: db,
 	}
 }
 
-// Try both methods of finding the first block, with the receipt method taking precedence
+// RetrieveFirstBlock fetches the block number for the earliest block in the db
+// Tries both methods of finding the first block, with the receipt method taking precedence
 func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
 	i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
 	if err != nil {
@@ -55,7 +58,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
 // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
 func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
 	var firstBlock int64
-	addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr)
+	addressID, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
 	if getAddressErr != nil {
 		return firstBlock, getAddressErr
 	}
@@ -66,16 +69,12 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
                            WHERE contract_address_id = $1
 		                   ORDER BY block_id ASC
 					       LIMIT 1)`,
-		addressId,
+		addressID,
 	)
 
 	return firstBlock, err
 }
 
-func addressRepository() repositories.AddressRepository {
-	return repositories.AddressRepository{}
-}
-
 // In which case this servers as a heuristic to find the first block by finding the first contract event log
 func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
 	var firstBlock int
@@ -88,7 +87,7 @@ func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64,
 	return int64(firstBlock), err
 }
 
-// Method to retrieve the most recent block in vDB
+// RetrieveMostRecentBlock retrieves the most recent block number in vDB
 func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
 	var lastBlock int64
 	err := r.db.Get(

pkg/contract_watcher/full/retriever/block_retriever_test.go

@@ -17,10 +17,11 @@
 package retriever_test
 
 import (
+	"strings"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"strings"
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"

pkg/contract_watcher/full/retriever/retriever_suite_test.go

@@ -17,12 +17,12 @@
 package retriever_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestRetriever(t *testing.T) {

pkg/contract_watcher/full/transformer/transformer.go

@@ -19,6 +19,8 @@ package transformer
 import (
 	"errors"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
@@ -33,6 +35,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
+// Transformer is the top level struct for transforming watched contract data
 // Requires a fully synced vDB and a running eth node (or infura)
 type Transformer struct {
 	// Database interfaces
@@ -58,7 +61,7 @@ type Transformer struct {
 	LastBlock int64
 }
 
-// Transformer takes in config for blockchain, database, and network id
+// NewTransformer takes in contract config, blockchain, and database, and returns a new Transformer
 func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer {
 	return &Transformer{
 		Poller:                     poller.NewPoller(BC, DB, types.FullSync),
@@ -73,6 +76,7 @@ func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.
 	}
 }
 
+// Init initializes the transformer
 // Use after creating and setting transformer
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
@@ -106,7 +110,11 @@ func (tr *Transformer) Init() error {
 
 		// Get contract name if it has one
 		var name = new(string)
-		tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock)
+		pollingErr := tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock)
+		if pollingErr != nil {
+			// can't return this error because "name" might not exist on the contract
+			logrus.Warnf("error fetching contract data: %s", pollingErr.Error())
+		}
 
 		// Remove any potential accidental duplicate inputs in arg filter values
 		eventArgs := map[string]bool{}
@@ -161,6 +169,7 @@ func (tr *Transformer) Init() error {
 	return nil
 }
 
+// Execute runs the transformation processes
 // Iterates through stored, initialized contract objects
 // Iterates through contract's event filters, grabbing watched event logs
 // Uses converter to convert logs into custom log type
@@ -221,6 +230,7 @@ func (tr *Transformer) Execute() error {
 	return nil
 }
 
+// GetConfig returns the transformers config; satisfies the transformer interface
 func (tr *Transformer) GetConfig() config.ContractConfig {
 	return tr.Config
 }

pkg/contract_watcher/full/transformer/transformer_suite_test.go

@@ -17,12 +17,12 @@
 package transformer_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestTransformer(t *testing.T) {

pkg/contract_watcher/header/converter/converter.go

@@ -18,7 +18,6 @@ package converter
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math/big"
 	"strconv"
@@ -32,16 +31,19 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
 )
 
+// ConverterInterface is the interface for converting geth logs to our custom log type
 type ConverterInterface interface {
 	Convert(logs []gethTypes.Log, event types.Event, headerID int64) ([]types.Log, error)
 	ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error)
 	Update(info *contract.Contract)
 }
 
+// Converter is the underlying struct for the ConverterInterface
 type Converter struct {
 	ContractInfo *contract.Contract
 }
 
+// Update is used to configure the converter with a specific contract
 func (c *Converter) Update(info *contract.Contract) {
 	c.ContractInfo = info
 }
@@ -98,7 +100,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 				strValues[fieldName] = converted.String()
 				seenHashes = append(seenHashes, converted)
 			default:
-				return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
+				return nil, fmt.Errorf("error: unhandled abi type %T", input)
 			}
 		}
 
@@ -114,7 +116,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 				Values:           strValues,
 				Raw:              raw,
 				TransactionIndex: log.TxIndex,
-				Id:               headerID,
+				ID:               headerID,
 			})
 
 			// Cache emitted values if their caching is turned on
@@ -130,9 +132,9 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in
 	return returnLogs, nil
 }
 
-// Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
+// ConvertBatch converts the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
 func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	eventsToLogs := make(map[string][]types.Log)
 	for _, event := range events {
 		eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@@ -141,7 +143,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
 			// If the log is of this event type, process it as such
 			if event.Sig() == log.Topics[0] {
 				values := make(map[string]interface{})
-				err := contract.UnpackLogIntoMap(values, event.Name, log)
+				err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 				if err != nil {
 					return nil, err
 				}
@@ -182,7 +184,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
 						strValues[fieldName] = converted.String()
 						seenHashes = append(seenHashes, converted)
 					default:
-						return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
+						return nil, fmt.Errorf("error: unhandled abi type %T", input)
 					}
 				}
 
@@ -198,7 +200,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
 						Values:           strValues,
 						Raw:              raw,
 						TransactionIndex: log.TxIndex,
-						Id:               headerID,
+						ID:               headerID,
 					})
 
 					// Cache emitted values that pass the argument filter if their caching is turned on

pkg/contract_watcher/header/converter/converter_test.go

@@ -72,11 +72,11 @@ var _ = Describe("Converter", func() {
 			Expect(logs[0].Values["to"]).To(Equal(sender1.String()))
 			Expect(logs[0].Values["from"]).To(Equal(sender2.String()))
 			Expect(logs[0].Values["value"]).To(Equal(value.String()))
-			Expect(logs[0].Id).To(Equal(int64(232)))
+			Expect(logs[0].ID).To(Equal(int64(232)))
 			Expect(logs[1].Values["to"]).To(Equal(sender2.String()))
 			Expect(logs[1].Values["from"]).To(Equal(sender1.String()))
 			Expect(logs[1].Values["value"]).To(Equal(value.String()))
-			Expect(logs[1].Id).To(Equal(int64(232)))
+			Expect(logs[1].ID).To(Equal(int64(232)))
 		})
 
 		It("Keeps track of addresses it sees if they will be used for method polling", func() {

pkg/contract_watcher/header/fetcher/fetcher.go

@@ -24,6 +24,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
+// Fetcher is the fetching interface
 type Fetcher interface {
 	FetchLogs(contractAddresses []string, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
 }
@@ -32,13 +33,14 @@ type fetcher struct {
 	blockChain core.BlockChain
 }
 
-func NewFetcher(blockchain core.BlockChain) *fetcher {
+// NewFetcher returns a new Fetcher
+func NewFetcher(blockchain core.BlockChain) Fetcher {
 	return &fetcher{
 		blockChain: blockchain,
 	}
 }
 
-// Checks all topic0s, on all addresses, fetching matching logs for the given header
+// FetchLogs checks all topic0s, on all addresses, fetching matching logs for the given header
 func (fetcher *fetcher) FetchLogs(contractAddresses []string, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
 	addresses := hexStringsToAddresses(contractAddresses)
 	blockHash := common.HexToHash(header.Hash)

pkg/contract_watcher/header/repository/header_repository.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"github.com/hashicorp/golang-lru"
-	"github.com/jmoiron/sqlx"
+	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -28,6 +28,7 @@ import (
 
 const columnCacheSize = 1000
 
+// HeaderRepository interfaces with the header and checked_headers tables
 type HeaderRepository interface {
 	AddCheckColumn(id string) error
 	AddCheckColumns(ids []string) error
@@ -45,7 +46,8 @@ type headerRepository struct {
 	columns *lru.Cache // Cache created columns to minimize db connections
 }
 
-func NewHeaderRepository(db *postgres.DB) *headerRepository {
+// NewHeaderRepository returns a new HeaderRepository
+func NewHeaderRepository(db *postgres.DB) HeaderRepository {
 	ccs, _ := lru.New(columnCacheSize)
 	return &headerRepository{
 		db:      db,
@@ -53,7 +55,7 @@ func NewHeaderRepository(db *postgres.DB) *headerRepository {
 	}
 }
 
-// Adds a checked_header column for the provided column id
+// AddCheckColumn adds a checked_header column for the provided column id
 func (r *headerRepository) AddCheckColumn(id string) error {
 	// Check cache to see if column already exists before querying pg
 	_, ok := r.columns.Get(id)
@@ -74,7 +76,7 @@ func (r *headerRepository) AddCheckColumn(id string) error {
 	return nil
 }
 
-// Adds a checked_header column for all of the provided column ids
+// AddCheckColumns adds a checked_header column for all of the provided column ids
 func (r *headerRepository) AddCheckColumns(ids []string) error {
 	var err error
 	baseQuery := "ALTER TABLE public.checked_headers"
@@ -98,7 +100,7 @@ func (r *headerRepository) AddCheckColumns(ids []string) error {
 	return err
 }
 
-// Marks the header checked for the provided column id
+// MarkHeaderChecked marks the header checked for the provided column id
 func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
 	_, err := r.db.Exec(`INSERT INTO public.checked_headers (header_id, `+id+`)
 		VALUES ($1, $2) 
@@ -107,7 +109,7 @@ func (r *headerRepository) MarkHeaderChecked(headerID int64, id string) error {
 	return err
 }
 
-// Marks the header checked for all of the provided column ids
+// MarkHeaderCheckedForAll marks the header checked for all of the provided column ids
 func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string) error {
 	pgStr := "INSERT INTO public.checked_headers (header_id, "
 	for _, id := range ids {
@@ -126,7 +128,7 @@ func (r *headerRepository) MarkHeaderCheckedForAll(headerID int64, ids []string)
 	return err
 }
 
-// Marks all of the provided headers checked for each of the provided column ids
+// MarkHeadersCheckedForAll marks all of the provided headers checked for each of the provided column ids
 func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids []string) error {
 	tx, err := r.db.Beginx()
 	if err != nil {
@@ -148,7 +150,10 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
 		pgStr = pgStr[:len(pgStr)-2]
 		_, err = tx.Exec(pgStr, header.Id)
 		if err != nil {
-			tx.Rollback()
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transaction: %s", rollbackErr.Error())
+			}
 			return err
 		}
 	}
@@ -156,7 +161,7 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
 	return err
 }
 
-// Returns missing headers for the provided checked_headers column id
+// MissingHeaders returns missing headers for the provided checked_headers column id
 func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64, id string) ([]core.Header, error) {
 	var result []core.Header
 	var query string
@@ -179,10 +184,10 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
 				ORDER BY headers.block_number`
 		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
 	}
-	return contiguousHeaders(result, startingBlockNumber), err
+	return continuousHeaders(result), err
 }
 
-// Returns missing headers for all of the provided checked_headers column ids
+// MissingHeadersForAll returns missing headers for all of the provided checked_headers column ids
 func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlockNumber int64, ids []string) ([]core.Header, error) {
 	var result []core.Header
 	var query string
@@ -207,29 +212,10 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock
 		query = baseQuery + endStr
 		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
 	}
-	return contiguousHeaders(result, startingBlockNumber), err
+	return continuousHeaders(result), err
 }
 
-// Returns a continuous set of headers that is contiguous with the provided startingBlockNumber
-func contiguousHeaders(headers []core.Header, startingBlockNumber int64) []core.Header {
-	if len(headers) < 1 {
-		return headers
-	}
-	previousHeader := headers[0].BlockNumber
-	if previousHeader != startingBlockNumber {
-		return []core.Header{}
-	}
-	for i := 1; i < len(headers); i++ {
-		previousHeader++
-		if headers[i].BlockNumber != previousHeader {
-			return headers[:i]
-		}
-	}
-
-	return headers
-}
-
-// Returns headers that have been checked for all of the provided event ids but not for the provided method ids
+// MissingMethodsCheckedEventsIntersection returns headers that have been checked for all of the provided event ids but not for the provided method ids
 func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlockNumber, endingBlockNumber int64, methodIds, eventIds []string) ([]core.Header, error) {
 	var result []core.Header
 	var query string
@@ -265,6 +251,7 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock
 // Returns a continuous set of headers
 func continuousHeaders(headers []core.Header) []core.Header {
 	if len(headers) < 1 {
+		logrus.Trace("no headers to arrange continuously")
 		return headers
 	}
 	previousHeader := headers[0].BlockNumber
@@ -278,16 +265,7 @@ func continuousHeaders(headers []core.Header) []core.Header {
 	return headers
 }
 
-// Check the repositories column id cache for a value
+// CheckCache checks the repositories column id cache for a value
 func (r *headerRepository) CheckCache(key string) (interface{}, bool) {
 	return r.columns.Get(key)
 }
-
-// Used to mark a header checked as part of some external transaction so as to group into one commit
-func (r *headerRepository) MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, eventID string) error {
-	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+eventID+`)
-		VALUES ($1, $2) 
-		ON CONFLICT (header_id) DO
-			UPDATE SET `+eventID+` = checked_headers.`+eventID+` + 1`, headerID, 1)
-	return err
-}

pkg/contract_watcher/header/repository/header_repository_test.go

@@ -18,7 +18,6 @@ package repository_test
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -26,6 +25,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
@@ -120,7 +120,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 		})
@@ -130,16 +130,16 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
 			h1 := missingHeaders[0]
 			h2 := missingHeaders[1]
 			h3 := missingHeaders[2]
-			Expect(h1.BlockNumber).To(Equal(int64(6194632)))
-			Expect(h2.BlockNumber).To(Equal(int64(6194633)))
-			Expect(h3.BlockNumber).To(Equal(int64(6194634)))
+			Expect(h1.BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
+			Expect(h2.BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
+			Expect(h3.BlockNumber).To(Equal(mocks.MockHeader3.BlockNumber))
 		})
 
 		It("Returns only contiguous chunks of headers", func() {
@@ -147,11 +147,11 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632)))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633)))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
 		})
 
 		It("Fails if eventID does not yet exist in check_headers table", func() {
@@ -159,7 +159,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			_, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, "notEventId")
+			_, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, "notEventId")
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -170,14 +170,14 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
 			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
@@ -186,7 +186,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194633, 6194635, eventIDs)
+			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
 		})
@@ -196,11 +196,23 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632)))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633)))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
+		})
+
+		It("returns headers after starting header if starting header not missing", func() {
+			addLaterHeaders(coreHeaderRepo)
+			err := contractHeaderRepo.AddCheckColumns(eventIDs)
+			Expect(err).NotTo(HaveOccurred())
+
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, -1, eventIDs)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(missingHeaders)).To(Equal(2))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader3.BlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader4.BlockNumber))
 		})
 
 		It("Fails if one of the eventIDs does not yet exist in check_headers table", func() {
@@ -209,7 +221,7 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			badEventIDs := append(eventIDs, "notEventId")
 
-			_, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, badEventIDs)
+			_, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, badEventIDs)
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -220,7 +232,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
@@ -228,7 +240,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0])
+			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
 		})
@@ -238,7 +250,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
@@ -246,7 +258,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderChecked(headerID, "notEventId")
 			Expect(err).To(HaveOccurred())
 
-			missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 		})
@@ -258,7 +270,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
@@ -266,7 +278,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 
-			missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0])
+			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
 		})
@@ -285,7 +297,7 @@ var _ = Describe("Repository", func() {
 			for _, id := range methodIDs {
 				err := contractHeaderRepo.AddCheckColumn(id)
 				Expect(err).ToNot(HaveOccurred())
-				missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id)
+				missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, id)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(len(missingHeaders)).To(Equal(3))
 			}
@@ -293,7 +305,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs)
 			Expect(err).ToNot(HaveOccurred())
 			for _, id := range methodIDs {
-				missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id)
+				missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, id)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(len(missingHeaders)).To(Equal(0))
 			}
@@ -310,7 +322,7 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 			}
 
-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 
@@ -325,23 +337,35 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 			}
 
-			intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(6194632, 6194635, methodIDs, eventIDs)
+			intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, methodIDs, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(intersectionHeaders)).To(Equal(1))
 			Expect(intersectionHeaders[0].Id).To(Equal(headerID2))
-
 		})
 	})
 })
 
 func addHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
+	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
+	Expect(err).NotTo(HaveOccurred())
 }
 
 func addDiscontinuousHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
+	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func addLaterHeaders(coreHeaderRepo repositories.HeaderRepository) {
+	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
+	Expect(err).NotTo(HaveOccurred())
 }

pkg/contract_watcher/header/repository/repository_suite_test.go

@@ -17,12 +17,12 @@
 package repository_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestRepository(t *testing.T) {

pkg/contract_watcher/header/retriever/block_retriever.go

@@ -20,7 +20,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-// Block retriever is used to retrieve the first block for a given contract and the most recent block
+// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
 // It requires a vDB synced database with blocks, transactions, receipts, and logs
 type BlockRetriever interface {
 	RetrieveFirstBlock() (int64, error)
@@ -31,13 +31,14 @@ type blockRetriever struct {
 	db *postgres.DB
 }
 
-func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
+// NewBlockRetriever returns a new BlockRetriever
+func NewBlockRetriever(db *postgres.DB) BlockRetriever {
 	return &blockRetriever{
 		db: db,
 	}
 }
 
-// Retrieve block number of earliest header in repo
+// RetrieveFirstBlock retrieves block number of earliest header in repo
 func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
 	var firstBlock int
 	err := r.db.Get(
@@ -48,7 +49,7 @@ func (r *blockRetriever) RetrieveFirstBlock() (int64, error) {
 	return int64(firstBlock), err
 }
 
-// Retrieve block number of latest header in repo
+// RetrieveMostRecentBlock retrieves block number of latest header in repo
 func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
 	var lastBlock int
 	err := r.db.Get(

pkg/contract_watcher/header/retriever/block_retriever_test.go

@@ -44,9 +44,12 @@ var _ = Describe("Block Retriever", func() {
 
 	Describe("RetrieveFirstBlock", func() {
 		It("Retrieves block number of earliest header in the database", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			_, err := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			Expect(err).ToNot(HaveOccurred())
 
 			i, err := r.RetrieveFirstBlock()
 			Expect(err).NotTo(HaveOccurred())
@@ -61,9 +64,12 @@ var _ = Describe("Block Retriever", func() {
 
 	Describe("RetrieveMostRecentBlock", func() {
 		It("Retrieves the latest header's block number", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			_, err := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			Expect(err).ToNot(HaveOccurred())
 
 			i, err := r.RetrieveMostRecentBlock()
 			Expect(err).ToNot(HaveOccurred())

pkg/contract_watcher/header/retriever/retriever_suite_test.go

@@ -17,12 +17,12 @@
 package retriever_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestRetriever(t *testing.T) {

pkg/contract_watcher/header/transformer/transformer.go

@@ -17,11 +17,14 @@
 package transformer
 
 import (
+	"database/sql"
 	"errors"
+	"fmt"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
@@ -37,6 +40,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+// Transformer is the top level struct for transforming watched contract data
 // Requires a header synced vDB (headers) and a running eth node (or infura)
 type Transformer struct {
 	// Database interfaces
@@ -73,7 +77,7 @@ type Transformer struct {
 // 3. Init
 // 4. Execute
 
-// Transformer takes in config for blockchain, database, and network id
+// NewTransformer takes in a contract config, blockchain, and database, and returns a new Transformer
 func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {
 
 	return &Transformer{
@@ -89,6 +93,7 @@ func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.
 	}
 }
 
+// Init initialized the Transformer
 // Use after creating and setting transformer
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
@@ -107,22 +112,27 @@ func (tr *Transformer) Init() error {
 		// Configure Abi
 		if tr.Config.Abis[contractAddr] == "" {
 			// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
-			err := tr.Parser.Parse(contractAddr)
-			if err != nil {
-				return err
+			parseErr := tr.Parser.Parse(contractAddr)
+			if parseErr != nil {
+				return fmt.Errorf("error parsing contract by address: %s", parseErr.Error())
 			}
 		} else {
 			// If we have an abi from the config, load that into the parser
-			err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
-			if err != nil {
-				return err
+			parseErr := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
+			if parseErr != nil {
+				return fmt.Errorf("error parsing contract abi: %s", parseErr.Error())
 			}
 		}
 
 		// Get first block and most recent block number in the header repo
-		firstBlock, err := tr.Retriever.RetrieveFirstBlock()
-		if err != nil {
-			return err
+		firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock()
+		if retrieveErr != nil {
+			if retrieveErr == sql.ErrNoRows {
+				logrus.Error(fmt.Errorf("error retrieving first block: %s", retrieveErr.Error()))
+				firstBlock = 0
+			} else {
+				return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error())
+			}
 		}
 
 		// Set to specified range if it falls within the bounds
@@ -132,7 +142,11 @@ func (tr *Transformer) Init() error {
 
 		// Get contract name if it has one
 		var name = new(string)
-		tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1)
+		pollingErr := tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1)
+		if pollingErr != nil {
+			// can't return this error because "name" might not exist on the contract
+			logrus.Warnf("error fetching contract data: %s", pollingErr.Error())
+		}
 
 		// Remove any potential accidental duplicate inputs
 		eventArgs := map[string]bool{}
@@ -164,14 +178,14 @@ func (tr *Transformer) Init() error {
 		// Create checked_headers columns for each event id and append to list of all event ids
 		tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events))
 		for _, event := range con.Events {
-			eventId := strings.ToLower(event.Name + "_" + con.Address)
-			err := tr.HeaderRepository.AddCheckColumn(eventId)
-			if err != nil {
-				return err
+			eventID := strings.ToLower(event.Name + "_" + con.Address)
+			addColumnErr := tr.HeaderRepository.AddCheckColumn(eventID)
+			if addColumnErr != nil {
+				return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
 			}
 			// Keep track of this event id; sorted and unsorted
-			tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventId)
-			tr.eventIds = append(tr.eventIds, eventId)
+			tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventID)
+			tr.eventIds = append(tr.eventIds, eventID)
 			// Append this event sig to the filters
 			tr.eventFilters = append(tr.eventFilters, event.Sig())
 		}
@@ -179,12 +193,12 @@ func (tr *Transformer) Init() error {
 		// Create checked_headers columns for each method id and append list of all method ids
 		tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods))
 		for _, m := range con.Methods {
-			methodId := strings.ToLower(m.Name + "_" + con.Address)
-			err := tr.HeaderRepository.AddCheckColumn(methodId)
-			if err != nil {
-				return err
+			methodID := strings.ToLower(m.Name + "_" + con.Address)
+			addColumnErr := tr.HeaderRepository.AddCheckColumn(methodID)
+			if addColumnErr != nil {
+				return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
 			}
-			tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodId)
+			tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodID)
 		}
 
 		// Update start to the lowest block
@@ -196,15 +210,16 @@ func (tr *Transformer) Init() error {
 	return nil
 }
 
+// Execute runs the transformation processes
 func (tr *Transformer) Execute() error {
 	if len(tr.Contracts) == 0 {
 		return errors.New("error: transformer has no initialized contracts")
 	}
 
 	// Find unchecked headers for all events across all contracts; these are returned in asc order
-	missingHeaders, err := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds)
-	if err != nil {
-		return err
+	missingHeaders, missingHeadersErr := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds)
+	if missingHeadersErr != nil {
+		return fmt.Errorf("error getting missing headers: %s", missingHeadersErr.Error())
 	}
 
 	// Iterate over headers
@@ -216,27 +231,27 @@ func (tr *Transformer) Execute() error {
 		// Map to sort batch fetched logs by which contract they belong to, for post fetch processing
 		sortedLogs := make(map[string][]gethTypes.Log)
 		// And fetch all event logs across contracts at this header
-		allLogs, err := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header)
-		if err != nil {
-			return err
+		allLogs, fetchErr := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header)
+		if fetchErr != nil {
+			return fmt.Errorf("error fetching logs: %s", fetchErr.Error())
 		}
 
 		// If no logs are found mark the header checked for all of these eventIDs
 		// and continue to method polling and onto the next iteration
 		if len(allLogs) < 1 {
-			err = tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
-			if err != nil {
-				return err
+			markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
+			if markCheckedErr != nil {
+				return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
 			}
-			err = tr.methodPolling(header, tr.sortedMethodIds)
-			if err != nil {
-				return err
+			pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
+			if pollingErr != nil {
+				return fmt.Errorf("error polling methods: %s", pollingErr.Error())
 			}
 			tr.Start = header.BlockNumber + 1 // Empty header; setup to start at the next header
+			logrus.Tracef("no logs found for block %d, continuing", header.BlockNumber)
 			continue
 		}
 
-		// Sort logs by the contract they belong to
 		for _, log := range allLogs {
 			addr := strings.ToLower(log.Address.Hex())
 			sortedLogs[addr] = append(sortedLogs[addr], log)
@@ -245,6 +260,7 @@ func (tr *Transformer) Execute() error {
 		// Process logs for each contract
 		for conAddr, logs := range sortedLogs {
 			if logs == nil {
+				logrus.Tracef("no logs found for contract %s at block %d, continuing", conAddr, header.BlockNumber)
 				continue
 			}
 			// Configure converter with this contract
@@ -252,34 +268,34 @@ func (tr *Transformer) Execute() error {
 			tr.Converter.Update(con)
 
 			// Convert logs into batches of log mappings (eventName => []types.Logs
-			convertedLogs, err := tr.Converter.ConvertBatch(logs, con.Events, header.Id)
-			if err != nil {
-				return err
+			convertedLogs, convertErr := tr.Converter.ConvertBatch(logs, con.Events, header.Id)
+			if convertErr != nil {
+				return fmt.Errorf("error converting logs: %s", convertErr.Error())
 			}
 			// Cycle through each type of event log and persist them
 			for eventName, logs := range convertedLogs {
 				// If logs for this event are empty, mark them checked at this header and continue
 				if len(logs) < 1 {
-					eventId := strings.ToLower(eventName + "_" + con.Address)
-					err = tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId)
-					if err != nil {
-						return err
-					}
+					logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber)
 					continue
 				}
 				// If logs aren't empty, persist them
-				// Header is marked checked in the transactions
-				err = tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
-				if err != nil {
-					return err
+				persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
+				if persistErr != nil {
+					return fmt.Errorf("error persisting logs: %s", persistErr.Error())
 				}
 			}
 		}
 
+		markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
+		if markCheckedErr != nil {
+			return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
+		}
+
 		// Poll contracts at this block height
-		err = tr.methodPolling(header, tr.sortedMethodIds)
-		if err != nil {
-			return err
+		pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
+		if pollingErr != nil {
+			return fmt.Errorf("error polling methods: %s", pollingErr.Error())
 		}
 		// Success; setup to start at the next header
 		tr.Start = header.BlockNumber + 1
@@ -294,25 +310,27 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str
 		// Skip method polling processes if no methods are specified
 		// Also don't try to poll methods below this contract's specified starting block
 		if len(con.Methods) == 0 || header.BlockNumber < con.StartingBlock {
+			logrus.Tracef("not polling contract: %s", con.Address)
 			continue
 		}
 
 		// Poll all methods for this contract at this header
-		err := tr.Poller.PollContractAt(*con, header.BlockNumber)
-		if err != nil {
-			return err
+		pollingErr := tr.Poller.PollContractAt(*con, header.BlockNumber)
+		if pollingErr != nil {
+			return fmt.Errorf("error polling contract %s: %s", con.Address, pollingErr.Error())
 		}
 
 		// Mark this header checked for the methods
-		err = tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, sortedMethodIds[con.Address])
-		if err != nil {
-			return err
+		markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, sortedMethodIds[con.Address])
+		if markCheckedErr != nil {
+			return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
 		}
 	}
 
 	return nil
 }
 
+// GetConfig returns the transformers config; satisfies the transformer interface
 func (tr *Transformer) GetConfig() config.ContractConfig {
 	return tr.Config
 }

pkg/contract_watcher/header/transformer/transformer_suite_test.go

@@ -17,12 +17,12 @@
 package transformer_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestTransformer(t *testing.T) {

pkg/contract_watcher/header/transformer/transformer_test.go

@@ -17,6 +17,8 @@
 package transformer_test
 
 import (
+	"database/sql"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -68,7 +70,7 @@ var _ = Describe("Transformer", func() {
 			err := t.Init()
 
 			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
+			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		})
 	})
 
@@ -101,7 +103,17 @@ var _ = Describe("Transformer", func() {
 			Expect(c.Address).To(Equal(fakeAddress))
 		})
 
-		It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
+		It("uses first block from config if vDB headers table has no rows", func() {
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
+			blockRetriever.FirstBlockErr = sql.ErrNoRows
+			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
+
+			err := t.Init()
+
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("returns error if fetching first block fails for other reason", func() {
 			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})
@@ -109,7 +121,7 @@ var _ = Describe("Transformer", func() {
 			err := t.Init()
 
 			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
+			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		})
 	})
 })

pkg/contract_watcher/shared/constants/interface.go

@@ -20,11 +20,10 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
-// Basic abi needed to check which interfaces are adhered to
+// SupportsInterfaceABI is the basic abi needed to check which interfaces are adhered to
 var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]`
 
 // Individual event interfaces for constructing ABI from
-var SupportsInterace = `{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}`
 var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}`
 var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}`
 var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}`
@@ -34,11 +33,10 @@ var TextChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"
 var MultihashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"MultihashChanged","type":"event"}`
 var ContenthashChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes"}],"name":"ContenthashChanged","type":"event"}`
 
-var StartingBlock = int64(3648359)
-
 // Resolver interface signatures
 type Interface int
 
+// Interface enums
 const (
 	MetaSig Interface = iota
 	AddrChangeSig
@@ -51,6 +49,7 @@ const (
 	ContentHashChangeSig
 )
 
+// Hex returns the hex signature for an interface
 func (e Interface) Hex() string {
 	strings := [...]string{
 		"0x01ffc9a7",
@@ -71,6 +70,7 @@ func (e Interface) Hex() string {
 	return strings[e]
 }
 
+// Bytes returns the bytes signature for an interface
 func (e Interface) Bytes() [4]uint8 {
 	if e < MetaSig || e > ContentHashChangeSig {
 		return [4]byte{}
@@ -86,6 +86,7 @@ func (e Interface) Bytes() [4]uint8 {
 	return byArray
 }
 
+// EventSig returns the event signature for an interface
 func (e Interface) EventSig() string {
 	strings := [...]string{
 		"",
@@ -106,6 +107,7 @@ func (e Interface) EventSig() string {
 	return strings[e]
 }
 
+// MethodSig returns the method signature for an interface
 func (e Interface) MethodSig() string {
 	strings := [...]string{
 		"supportsInterface(bytes4)",

pkg/contract_watcher/shared/contract/contract.go

@@ -48,6 +48,7 @@ type Contract struct {
 	Piping         bool                         // Whether or not to pipe method results forward as arguments to subsequent methods
 }
 
+// Init initializes a contract object
 // If we will be calling methods that use addr, hash, or byte arrays
 // as arguments then we initialize maps to hold these types of values
 func (c Contract) Init() *Contract {
@@ -66,7 +67,7 @@ func (c Contract) Init() *Contract {
 	return &c
 }
 
-// Use contract info to generate event filters - full sync contract watcher only
+// GenerateFilters uses contract info to generate event filters - full sync contract watcher only
 func (c *Contract) GenerateFilters() error {
 	c.Filters = map[string]filters.LogFilter{}
 
@@ -87,7 +88,7 @@ func (c *Contract) GenerateFilters() error {
 	return nil
 }
 
-// Returns true if address is in list of arguments to
+// WantedEventArg returns true if address is in list of arguments to
 // filter events for or if no filtering is specified
 func (c *Contract) WantedEventArg(arg string) bool {
 	if c.FilterArgs == nil {
@@ -101,7 +102,7 @@ func (c *Contract) WantedEventArg(arg string) bool {
 	return false
 }
 
-// Returns true if address is in list of arguments to
+// WantedMethodArg returns true if address is in list of arguments to
 // poll methods with or if no filtering is specified
 func (c *Contract) WantedMethodArg(arg interface{}) bool {
 	if c.MethodArgs == nil {
@@ -121,7 +122,7 @@ func (c *Contract) WantedMethodArg(arg interface{}) bool {
 	return false
 }
 
-// Returns true if any mapping value matches filtered for address or if no filter exists
+// PassesEventFilter returns true if any mapping value matches filtered for address or if no filter exists
 // Used to check if an event log name-value mapping should be filtered or not
 func (c *Contract) PassesEventFilter(args map[string]string) bool {
 	for _, arg := range args {
@@ -133,7 +134,7 @@ func (c *Contract) PassesEventFilter(args map[string]string) bool {
 	return false
 }
 
-// Add event emitted address to our list if it passes filter and method polling is on
+// AddEmittedAddr adds event emitted addresses to our list if it passes filter and method polling is on
 func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
 	for _, addr := range addresses {
 		if c.WantedMethodArg(addr) && c.Methods != nil {
@@ -142,7 +143,7 @@ func (c *Contract) AddEmittedAddr(addresses ...interface{}) {
 	}
 }
 
-// Add event emitted hash to our list if it passes filter and method polling is on
+// AddEmittedHash adds event emitted hashes to our list if it passes filter and method polling is on
 func (c *Contract) AddEmittedHash(hashes ...interface{}) {
 	for _, hash := range hashes {
 		if c.WantedMethodArg(hash) && c.Methods != nil {
@@ -151,6 +152,7 @@ func (c *Contract) AddEmittedHash(hashes ...interface{}) {
 	}
 }
 
+// StringifyArg resolves a method argument type to string type
 func StringifyArg(arg interface{}) (str string) {
 	switch arg.(type) {
 	case string:

pkg/contract_watcher/shared/fetcher/fetcher.go

@@ -29,7 +29,7 @@ import (
 // Fetcher serves as the lower level data fetcher that calls the underlying
 // blockchain's FetchConctractData method for a given return type
 
-// Interface definition for a Fetcher
+// FetcherInterface is the interface definition for a fetcher
 type FetcherInterface interface {
 	FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error)
 	FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error)
@@ -47,7 +47,7 @@ func newFetcherError(err error, fetchMethod string) *fetcherError {
 
 // Fetcher struct
 type Fetcher struct {
-	BlockChain core.BlockChain // Underyling Blockchain
+	BlockChain core.BlockChain // Underlying Blockchain
 }
 
 // Fetcher error
@@ -56,14 +56,14 @@ type fetcherError struct {
 	fetchMethod string
 }
 
-// Fetcher error method
+// Error method
 func (fe *fetcherError) Error() string {
 	return fmt.Sprintf("Error fetching %s: %s", fe.fetchMethod, fe.err)
 }
 
 //  Generic Fetcher methods used by Getters to call contract methods
 
-// Method used to fetch big.Int value from contract
+// FetchBigInt is the method used to fetch big.Int value from contract
 func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (big.Int, error) {
 	var result = new(big.Int)
 	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@@ -75,7 +75,7 @@ func (f Fetcher) FetchBigInt(method, contractAbi, contractAddress string, blockN
 	return *result, nil
 }
 
-// Method used to fetch bool value from contract
+// FetchBool is the method used to fetch bool value from contract
 func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (bool, error) {
 	var result = new(bool)
 	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@@ -87,7 +87,7 @@ func (f Fetcher) FetchBool(method, contractAbi, contractAddress string, blockNum
 	return *result, nil
 }
 
-// Method used to fetch address value from contract
+// FetchAddress is the method used to fetch address value from contract
 func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Address, error) {
 	var result = new(common.Address)
 	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@@ -99,7 +99,7 @@ func (f Fetcher) FetchAddress(method, contractAbi, contractAddress string, block
 	return *result, nil
 }
 
-// Method used to fetch string value from contract
+// FetchString is the method used to fetch string value from contract
 func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (string, error) {
 	var result = new(string)
 	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)
@@ -111,7 +111,7 @@ func (f Fetcher) FetchString(method, contractAbi, contractAddress string, blockN
 	return *result, nil
 }
 
-// Method used to fetch hash value from contract
+// FetchHash is the method used to fetch hash value from contract
 func (f Fetcher) FetchHash(method, contractAbi, contractAddress string, blockNumber int64, methodArgs []interface{}) (common.Hash, error) {
 	var result = new(common.Hash)
 	err := f.BlockChain.FetchContractData(contractAbi, contractAddress, method, methodArgs, &result, blockNumber)

pkg/contract_watcher/shared/getter/getter_test.go

@@ -24,10 +24,10 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/getter"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -45,11 +45,12 @@ var _ = Describe("Interface Getter", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			interfaceGetter := getter.NewInterfaceGetter(blockChain)
-			abi := interfaceGetter.GetABI(constants.PublicResolverAddress, blockNumber)
+			abi, err := interfaceGetter.GetABI(constants.PublicResolverAddress, blockNumber)
+			Expect(err).NotTo(HaveOccurred())
 			Expect(abi).To(Equal(expectedABI))
-			_, err = geth.ParseAbi(abi)
+			_, err = eth.ParseAbi(abi)
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})

pkg/contract_watcher/shared/getter/interface_getter.go

@@ -17,13 +17,16 @@
 package getter
 
 import (
+	"fmt"
+
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
+// InterfaceGetter is used to derive the interface of a contract
 type InterfaceGetter interface {
-	GetABI(resolverAddr string, blockNumber int64) string
+	GetABI(resolverAddr string, blockNumber int64) (string, error)
 	GetBlockChain() core.BlockChain
 }
 
@@ -31,7 +34,8 @@ type interfaceGetter struct {
 	fetcher.Fetcher
 }
 
-func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter {
+// NewInterfaceGetter returns a new InterfaceGetter
+func NewInterfaceGetter(blockChain core.BlockChain) InterfaceGetter {
 	return &interfaceGetter{
 		Fetcher: fetcher.Fetcher{
 			BlockChain: blockChain,
@@ -39,15 +43,19 @@ func NewInterfaceGetter(blockChain core.BlockChain) *interfaceGetter {
 	}
 }
 
-// Used to construct a custom ABI based on the results from calling supportsInterface
-func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) string {
+// GetABI is used to construct a custom ABI based on the results from calling supportsInterface
+func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) (string, error) {
 	a := constants.SupportsInterfaceABI
 	args := make([]interface{}, 1)
 	args[0] = constants.MetaSig.Bytes()
 	supports, err := g.getSupportsInterface(a, resolverAddr, blockNumber, args)
-	if err != nil || !supports {
-		return ""
+	if err != nil {
+		return "", fmt.Errorf("call to getSupportsInterface failed: %v", err)
 	}
+	if !supports {
+		return "", fmt.Errorf("contract does not support interface")
+	}
+
 	abiStr := `[`
 	args[0] = constants.AddrChangeSig.Bytes()
 	supports, err = g.getSupportsInterface(a, resolverAddr, blockNumber, args)
@@ -91,7 +99,7 @@ func (g *interfaceGetter) GetABI(resolverAddr string, blockNumber int64) string
 	}
 	abiStr = abiStr[:len(abiStr)-1] + `]`
 
-	return abiStr
+	return abiStr, nil
 }
 
 // Use this method to check whether or not a contract supports a given method/event interface
@@ -99,7 +107,7 @@ func (g *interfaceGetter) getSupportsInterface(contractAbi, contractAddress stri
 	return g.Fetcher.FetchBool("supportsInterface", contractAbi, contractAddress, blockNumber, methodArgs)
 }
 
-// Method to retrieve the Getter's blockchain
+// GetBlockChain is a method to retrieve the Getter's blockchain
 func (g *interfaceGetter) GetBlockChain() core.BlockChain {
 	return g.Fetcher.BlockChain
 }

pkg/contract_watcher/shared/helpers/helpers.go

@@ -27,6 +27,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
+// ConvertToLog converts a watched event to a log
 func ConvertToLog(watchedEvent core.WatchedEvent) types.Log {
 	allTopics := []string{watchedEvent.Topic0, watchedEvent.Topic1, watchedEvent.Topic2, watchedEvent.Topic3}
 	var nonNilTopics []string
@@ -56,12 +57,14 @@ func createTopics(topics ...string) []common.Hash {
 	return topicsArray
 }
 
+// BigFromString creates a big.Int from a string
 func BigFromString(n string) *big.Int {
 	b := new(big.Int)
 	b.SetString(n, 10)
 	return b
 }
 
+// GenerateSignature returns the keccak256 hash hex of a string
 func GenerateSignature(s string) string {
 	eventSignature := []byte(s)
 	hash := crypto.Keccak256Hash(eventSignature)

pkg/contract_watcher/shared/helpers/test_helpers/database.go

@@ -30,10 +30,10 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -115,9 +115,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	madeNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := eth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter)
 
 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",
@@ -294,8 +294,7 @@ func TearDown(db *postgres.DB) {
 
 	_, err = tx.Exec(`CREATE TABLE checked_headers (
     	id SERIAL PRIMARY KEY,
-    	header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-    	check_count INTEGER NOT NULL DEFAULT 1);`)
+    	header_id INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE);`)
 	Expect(err).NotTo(HaveOccurred())
 
 	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)

pkg/contract_watcher/shared/helpers/test_helpers/mocks/entities.go

@@ -324,7 +324,7 @@ var MockConfig = config.ContractConfig{
 		"0x1234567890abcdef": "fake_abi",
 	},
 	Events: map[string][]string{
-		"0x1234567890abcdef": []string{"Transfer"},
+		"0x1234567890abcdef": {"Transfer"},
 	},
 	Methods: map[string][]string{
 		"0x1234567890abcdef": nil,

pkg/contract_watcher/shared/helpers/test_helpers/mocks/parser.go

@@ -20,7 +20,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi"
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 )
 
 // Mock parser
@@ -50,7 +50,7 @@ func (p *parser) ParsedAbi() abi.ABI {
 // for the given contract address
 func (p *parser) Parse() error {
 	var err error
-	p.parsedAbi, err = geth.ParseAbi(p.abi)
+	p.parsedAbi, err = eth.ParseAbi(p.abi)
 
 	return err
 }

pkg/contract_watcher/shared/helpers/test_helpers/test_data.go

@@ -35,7 +35,7 @@ var TusdConfig = config.ContractConfig{
 		tusd: "",
 	},
 	Events: map[string][]string{
-		tusd: []string{"Transfer"},
+		tusd: {"Transfer"},
 	},
 	Methods: map[string][]string{
 		tusd: nil,
@@ -60,7 +60,7 @@ var ENSConfig = config.ContractConfig{
 		ens: "",
 	},
 	Events: map[string][]string{
-		ens: []string{"NewOwner"},
+		ens: {"NewOwner"},
 	},
 	Methods: map[string][]string{
 		ens: nil,
@@ -87,8 +87,8 @@ var ENSandTusdConfig = config.ContractConfig{
 		tusd: "",
 	},
 	Events: map[string][]string{
-		ens:  []string{"NewOwner"},
-		tusd: []string{"Transfer"},
+		ens:  {"NewOwner"},
+		tusd: {"Transfer"},
 	},
 	Methods: map[string][]string{
 		ens:  nil,

pkg/contract_watcher/shared/parser/parser.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 )
 
 // Parser is used to fetch and parse contract ABIs
@@ -40,28 +40,31 @@ type Parser interface {
 }
 
 type parser struct {
-	client    *geth.EtherScanAPI
+	client    *eth.EtherScanAPI
 	abi       string
 	parsedAbi abi.ABI
 }
 
-func NewParser(network string) *parser {
-	url := geth.GenURL(network)
+// NewParser returns a new Parser
+func NewParser(network string) Parser {
+	url := eth.GenURL(network)
 
 	return &parser{
-		client: geth.NewEtherScanClient(url),
+		client: eth.NewEtherScanClient(url),
 	}
 }
 
+// Abi returns the parser's configured abi string
 func (p *parser) Abi() string {
 	return p.abi
 }
 
+// ParsedAbi returns the parser's parsed abi
 func (p *parser) ParsedAbi() abi.ABI {
 	return p.parsedAbi
 }
 
-// Retrieves and parses the abi string
+// Parse retrieves and parses the abi string
 // for the given contract address
 func (p *parser) Parse(contractAddr string) error {
 	// If the abi is one our locally stored abis, fetch
@@ -69,7 +72,7 @@ func (p *parser) Parse(contractAddr string) error {
 	knownAbi, err := p.lookUp(contractAddr)
 	if err == nil {
 		p.abi = knownAbi
-		p.parsedAbi, err = geth.ParseAbi(knownAbi)
+		p.parsedAbi, err = eth.ParseAbi(knownAbi)
 		return err
 	}
 	// Try getting abi from etherscan
@@ -79,29 +82,29 @@ func (p *parser) Parse(contractAddr string) error {
 	}
 	//TODO: Implement other ways to fetch abi
 	p.abi = abiStr
-	p.parsedAbi, err = geth.ParseAbi(abiStr)
+	p.parsedAbi, err = eth.ParseAbi(abiStr)
 
 	return err
 }
 
-// Loads and parses an abi from a given abi string
+// ParseAbiStr loads and parses an abi from a given abi string
 func (p *parser) ParseAbiStr(abiStr string) error {
 	var err error
 	p.abi = abiStr
-	p.parsedAbi, err = geth.ParseAbi(abiStr)
+	p.parsedAbi, err = eth.ParseAbi(abiStr)
 
 	return err
 }
 
 func (p *parser) lookUp(contractAddr string) (string, error) {
-	if v, ok := constants.Abis[common.HexToAddress(contractAddr)]; ok {
+	if v, ok := constants.ABIs[common.HexToAddress(contractAddr)]; ok {
 		return v, nil
 	}
 
 	return "", errors.New("ABI not present in lookup table")
 }
 
-// Returns only specified methods, if they meet the criteria
+// GetSelectMethods returns only specified methods, if they meet the criteria
 // Returns as array with methods in same order they were specified
 // Nil or empty wanted array => no events are returned
 func (p *parser) GetSelectMethods(wanted []string) []types.Method {
@@ -121,7 +124,7 @@ func (p *parser) GetSelectMethods(wanted []string) []types.Method {
 	return methods
 }
 
-// Returns wanted methods
+// GetMethods returns wanted methods
 // Empty wanted array => all methods are returned
 // Nil wanted array => no methods are returned
 func (p *parser) GetMethods(wanted []string) []types.Method {
@@ -139,7 +142,7 @@ func (p *parser) GetMethods(wanted []string) []types.Method {
 	return methods
 }
 
-// Returns wanted events as map of types.Events
+// GetEvents returns wanted events as map of types.Events
 // Empty wanted array => all events are returned
 // Nil wanted array => no events are returned
 func (p *parser) GetEvents(wanted []string) map[string]types.Event {

pkg/contract_watcher/shared/parser/parser_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 )
 
 var _ = Describe("Parser", func() {
@@ -44,7 +44,7 @@ var _ = Describe("Parser", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			parsedAbi := mp.ParsedAbi()
-			expectedAbi, err := geth.ParseAbi(constants.DaiAbiString)
+			expectedAbi, err := eth.ParseAbi(constants.DaiAbiString)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(parsedAbi).To(Equal(expectedAbi))
 
@@ -73,7 +73,7 @@ var _ = Describe("Parser", func() {
 			expectedAbi := constants.DaiAbiString
 			Expect(p.Abi()).To(Equal(expectedAbi))
 
-			expectedParsedAbi, err := geth.ParseAbi(expectedAbi)
+			expectedParsedAbi, err := eth.ParseAbi(expectedAbi)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p.ParsedAbi()).To(Equal(expectedParsedAbi))
 		})

pkg/contract_watcher/shared/poller/poller.go

@@ -33,6 +33,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+// Poller is the interface for polling public contract methods
 type Poller interface {
 	PollContract(con contract.Contract, lastBlock int64) error
 	PollContractAt(con contract.Contract, blockNumber int64) error
@@ -45,13 +46,15 @@ type poller struct {
 	contract contract.Contract
 }
 
-func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) *poller {
+// NewPoller returns a new Poller
+func NewPoller(blockChain core.BlockChain, db *postgres.DB, mode types.Mode) Poller {
 	return &poller{
 		MethodRepository: repository.NewMethodRepository(db, mode),
 		bc:               blockChain,
 	}
 }
 
+// PollContract polls a contract's public methods from the contracts starting block to specified last block
 func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
 	for i := con.StartingBlock; i <= lastBlock; i++ {
 		if err := p.PollContractAt(con, i); err != nil {
@@ -62,6 +65,7 @@ func (p *poller) PollContract(con contract.Contract, lastBlock int64) error {
 	return nil
 }
 
+// PollContractAt polls a contract's public getter methods at the specified block height
 func (p *poller) PollContractAt(con contract.Contract, blockNumber int64) error {
 	p.contract = con
 	for _, m := range con.Methods {
@@ -98,7 +102,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
 	var out interface{}
 	err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, nil, &out, bn)
 	if err != nil {
-		return errors.New(fmt.Sprintf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
+		return fmt.Errorf("poller error calling 0 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
 	}
 	strOut, err := stringify(out)
 	if err != nil {
@@ -112,7 +116,7 @@ func (p *poller) pollNoArgAt(m types.Method, bn int64) error {
 	// Persist result immediately
 	err = p.PersistResults([]types.Result{result}, m, p.contract.Address, p.contract.Name)
 	if err != nil {
-		return errors.New(fmt.Sprintf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
+		return fmt.Errorf("poller error persisting 0 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
 	}
 
 	return nil
@@ -148,7 +152,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
 		var out interface{}
 		err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
 		if err != nil {
-			return errors.New(fmt.Sprintf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
+			return fmt.Errorf("poller error calling 1 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
 		}
 		strOut, err := stringify(out)
 		if err != nil {
@@ -164,7 +168,7 @@ func (p *poller) pollSingleArgAt(m types.Method, bn int64) error {
 	// Persist result set as batch
 	err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
 	if err != nil {
-		return errors.New(fmt.Sprintf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
+		return fmt.Errorf("poller error persisting 1 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
 	}
 
 	return nil
@@ -212,7 +216,7 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
 			var out interface{}
 			err := p.bc.FetchContractData(p.contract.Abi, p.contract.Address, m.Name, in, &out, bn)
 			if err != nil {
-				return errors.New(fmt.Sprintf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
+				return fmt.Errorf("poller error calling 2 argument method\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
 			}
 			strOut, err := stringify(out)
 			if err != nil {
@@ -228,13 +232,13 @@ func (p *poller) pollDoubleArgAt(m types.Method, bn int64) error {
 
 	err := p.PersistResults(results, m, p.contract.Address, p.contract.Name)
 	if err != nil {
-		return errors.New(fmt.Sprintf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err))
+		return fmt.Errorf("poller error persisting 2 argument method result\r\nblock: %d, method: %s, contract: %s\r\nerr: %v", bn, m.Name, p.contract.Address, err)
 	}
 
 	return nil
 }
 
-// This is just a wrapper around the poller blockchain's FetchContractData method
+// FetchContractData is just a wrapper around the poller blockchain's FetchContractData method
 func (p *poller) FetchContractData(contractAbi, contractAddress, method string, methodArgs []interface{}, result interface{}, blockNumber int64) error {
 	return p.bc.FetchContractData(contractAbi, contractAddress, method, methodArgs, result, blockNumber)
 }

pkg/contract_watcher/shared/repository/event_repository.go

@@ -22,8 +22,8 @@ import (
 	"strings"
 
 	"github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
@@ -31,10 +31,10 @@ import (
 const (
 	// Number of contract address and method ids to keep in cache
 	contractCacheSize = 100
-	eventChacheSize   = 1000
+	eventCacheSize    = 1000
 )
 
-// Event repository is used to persist event data into custom tables
+// EventRepository is used to persist event data into custom tables
 type EventRepository interface {
 	PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error
 	CreateEventTable(contractAddr string, event types.Event) (bool, error)
@@ -50,9 +50,10 @@ type eventRepository struct {
 	tables  *lru.Cache // Cache names of recently used tables to minimize db connections
 }
 
-func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
+// NewEventRepository returns a new EventRepository
+func NewEventRepository(db *postgres.DB, mode types.Mode) EventRepository {
 	ccs, _ := lru.New(contractCacheSize)
-	ecs, _ := lru.New(eventChacheSize)
+	ecs, _ := lru.New(eventCacheSize)
 	return &eventRepository{
 		db:      db,
 		mode:    mode,
@@ -61,21 +62,21 @@ func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
 	}
 }
 
-// Creates a schema for the contract if needed
+// PersistLogs creates a schema for the contract if needed
 // Creates table for the watched contract event if needed
 // Persists converted event log data into this custom table
 func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
 	if len(logs) == 0 {
 		return errors.New("event repository error: passed empty logs slice")
 	}
-	_, err := r.CreateContractSchema(contractAddr)
-	if err != nil {
-		return err
+	_, schemaErr := r.CreateContractSchema(contractAddr)
+	if schemaErr != nil {
+		return fmt.Errorf("error creating schema for contract %s: %s", contractAddr, schemaErr.Error())
 	}
 
-	_, err = r.CreateEventTable(contractAddr, eventInfo)
-	if err != nil {
-		return err
+	_, tableErr := r.CreateEventTable(contractAddr, eventInfo)
+	if tableErr != nil {
+		return fmt.Errorf("error creating table for event %s on contract %s: %s", eventInfo.Name, contractAddr, tableErr.Error())
 	}
 
 	return r.persistLogs(logs, eventInfo, contractAddr, contractName)
@@ -97,9 +98,9 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
 
 // Creates a custom postgres command to persist logs for the given event (compatible with header synced vDB)
 func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Beginx()
-	if err != nil {
-		return err
+	tx, txErr := r.db.Beginx()
+	if txErr != nil {
+		return fmt.Errorf("error beginning db transaction: %s", txErr.Error())
 	}
 
 	for _, event := range logs {
@@ -111,7 +112,7 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
 		// Preallocate slice of needed capacity and proceed to pack variables into it in same order they appear in string
 		data := make([]interface{}, 0, 5+el)
 		data = append(data,
-			event.Id,
+			event.ID,
 			contractName,
 			event.Raw,
 			event.LogIndex,
@@ -130,30 +131,26 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
 		}
 		pgStr = pgStr + ") ON CONFLICT DO NOTHING"
 
+		logrus.Tracef("query for inserting log: %s", pgStr)
 		// Add this query to the transaction
-		_, err = tx.Exec(pgStr, data...)
-		if err != nil {
-			tx.Rollback()
-			return err
+		_, execErr := tx.Exec(pgStr, data...)
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transactions while persisting logs: %s", rollbackErr.Error())
+			}
+			return fmt.Errorf("error executing query: %s", execErr.Error())
 		}
 	}
 
-	// Mark header as checked for this eventId
-	eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr)
-	err = repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
-	if err != nil {
-		tx.Rollback()
-		return err
-	}
-
 	return tx.Commit()
 }
 
 // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
 func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Beginx()
-	if err != nil {
-		return err
+	tx, txErr := r.db.Beginx()
+	if txErr != nil {
+		return fmt.Errorf("error beginning db transaction: %s", txErr.Error())
 	}
 
 	for _, event := range logs {
@@ -163,7 +160,7 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
 
 		data := make([]interface{}, 0, 4+el)
 		data = append(data,
-			event.Id,
+			event.ID,
 			contractName,
 			event.Block,
 			event.Tx)
@@ -179,17 +176,21 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
 		}
 		pgStr = pgStr + ") ON CONFLICT (vulcanize_log_id) DO NOTHING"
 
-		_, err = tx.Exec(pgStr, data...)
-		if err != nil {
-			tx.Rollback()
-			return err
+		logrus.Tracef("query for inserting log: %s", pgStr)
+		_, execErr := tx.Exec(pgStr, data...)
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transactions while persisting logs: %s", rollbackErr.Error())
+			}
+			return fmt.Errorf("error executing query: %s", execErr.Error())
 		}
 	}
 
 	return tx.Commit()
 }
 
-// Checks for event table and creates it if it does not already exist
+// CreateEventTable checks for event table and creates it if it does not already exist
 // Returns true if it created a new table; returns false if table already existed
 func (r *eventRepository) CreateEventTable(contractAddr string, event types.Event) (bool, error) {
 	tableID := fmt.Sprintf("%s_%s.%s_event", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(event.Name))
@@ -198,15 +199,15 @@ func (r *eventRepository) CreateEventTable(contractAddr string, event types.Even
 	if ok {
 		return false, nil
 	}
-	tableExists, err := r.checkForTable(contractAddr, event.Name)
-	if err != nil {
-		return false, err
+	tableExists, checkTableErr := r.checkForTable(contractAddr, event.Name)
+	if checkTableErr != nil {
+		return false, fmt.Errorf("error checking for table: %s", checkTableErr)
 	}
 
 	if !tableExists {
-		err = r.newEventTable(tableID, event)
-		if err != nil {
-			return false, err
+		createTableErr := r.newEventTable(tableID, event)
+		if createTableErr != nil {
+			return false, fmt.Errorf("error creating table: %s", createTableErr.Error())
 		}
 	}
 
@@ -258,7 +259,7 @@ func (r *eventRepository) checkForTable(contractAddr string, eventName string) (
 	return exists, err
 }
 
-// Checks for contract schema and creates it if it does not already exist
+// CreateContractSchema checks for contract schema and creates it if it does not already exist
 // Returns true if it created a new schema; returns false if schema already existed
 func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error) {
 	if contractAddr == "" {
@@ -270,14 +271,14 @@ func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error
 	if ok {
 		return false, nil
 	}
-	schemaExists, err := r.checkForSchema(contractAddr)
-	if err != nil {
-		return false, err
+	schemaExists, checkSchemaErr := r.checkForSchema(contractAddr)
+	if checkSchemaErr != nil {
+		return false, fmt.Errorf("error checking for schema: %s", checkSchemaErr.Error())
 	}
 	if !schemaExists {
-		err = r.newContractSchema(contractAddr)
-		if err != nil {
-			return false, err
+		createSchemaErr := r.newContractSchema(contractAddr)
+		if createSchemaErr != nil {
+			return false, fmt.Errorf("error creating schema: %s", createSchemaErr.Error())
 		}
 	}
 
@@ -304,10 +305,12 @@ func (r *eventRepository) checkForSchema(contractAddr string) (bool, error) {
 	return exists, err
 }
 
+// CheckSchemaCache is used to query the schema name cache
 func (r *eventRepository) CheckSchemaCache(key string) (interface{}, bool) {
 	return r.schemas.Get(key)
 }
 
+// CheckTableCache is used to query the table name cache
 func (r *eventRepository) CheckTableCache(key string) (interface{}, bool) {
 	return r.tables.Get(key)
 }

pkg/contract_watcher/shared/repository/event_repository_test.go

@@ -190,7 +190,7 @@ var _ = Describe("Repository", func() {
 				}
 				Expect(scanLog).To(Equal(expectedLog))
 
-				// Attempt to persist the same log again in seperate call
+				// Attempt to persist the same log again in separate call
 				err = dataStore.PersistLogs([]types.Log{*log}, event, con.Address, con.Name)
 				Expect(err).ToNot(HaveOccurred())
 
@@ -355,11 +355,6 @@ var _ = Describe("Repository", func() {
 				Expect(count).To(Equal(2))
 			})
 
-			It("Fails if the persisted event does not have a corresponding eventID column in the checked_headers table", func() {
-				err = dataStore.PersistLogs(logs, event, con.Address, con.Name)
-				Expect(err).To(HaveOccurred())
-			})
-
 			It("Fails with empty log", func() {
 				err = dataStore.PersistLogs([]types.Log{}, event, con.Address, con.Name)
 				Expect(err).To(HaveOccurred())

pkg/contract_watcher/shared/repository/method_repository.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -29,6 +30,7 @@ import (
 
 const methodCacheSize = 1000
 
+// MethodRepository is used to persist public getter method data
 type MethodRepository interface {
 	PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error
 	CreateMethodTable(contractAddr string, method types.Method) (bool, error)
@@ -44,7 +46,8 @@ type methodRepository struct {
 	tables  *lru.Cache // Cache names of recently used tables to minimize db connections
 }
 
-func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository {
+// NewMethodRepository returns a new MethodRepository
+func NewMethodRepository(db *postgres.DB, mode types.Mode) MethodRepository {
 	ccs, _ := lru.New(contractCacheSize)
 	mcs, _ := lru.New(methodCacheSize)
 	return &methodRepository{
@@ -55,7 +58,7 @@ func NewMethodRepository(db *postgres.DB, mode types.Mode) *methodRepository {
 	}
 }
 
-// Creates a schema for the contract if needed
+// PersistResults creates a schema for the contract if needed
 // Creates table for the contract method if needed
 // Persists method polling data into this custom table
 func (r *methodRepository) PersistResults(results []types.Result, methodInfo types.Method, contractAddr, contractName string) error {
@@ -112,7 +115,10 @@ func (r *methodRepository) persistResults(results []types.Result, methodInfo typ
 		// Add this query to the transaction
 		_, err = tx.Exec(pgStr, data...)
 		if err != nil {
-			tx.Rollback()
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transaction: %s", rollbackErr.Error())
+			}
 			return err
 		}
 	}
@@ -120,7 +126,7 @@ func (r *methodRepository) persistResults(results []types.Result, methodInfo typ
 	return tx.Commit()
 }
 
-// Checks for event table and creates it if it does not already exist
+// CreateMethodTable checks for event table and creates it if it does not already exist
 func (r *methodRepository) CreateMethodTable(contractAddr string, method types.Method) (bool, error) {
 	tableID := fmt.Sprintf("%s_%s.%s_method", r.mode.String(), strings.ToLower(contractAddr), strings.ToLower(method.Name))
 
@@ -173,7 +179,7 @@ func (r *methodRepository) checkForTable(contractAddr string, methodName string)
 	return exists, err
 }
 
-// Checks for contract schema and creates it if it does not already exist
+// CreateContractSchema checks for contract schema and creates it if it does not already exist
 func (r *methodRepository) CreateContractSchema(contractAddr string) (bool, error) {
 	if contractAddr == "" {
 		return false, errors.New("error: no contract address specified")
@@ -218,10 +224,12 @@ func (r *methodRepository) checkForSchema(contractAddr string) (bool, error) {
 	return exists, err
 }
 
+// CheckSchemaCache is used to query the schema name cache
 func (r *methodRepository) CheckSchemaCache(key string) (interface{}, bool) {
 	return r.schemas.Get(key)
 }
 
+// CheckTableCache is used to query the table name cache
 func (r *methodRepository) CheckTableCache(key string) (interface{}, bool) {
 	return r.tables.Get(key)
 }

pkg/contract_watcher/shared/repository/repository_suite_test.go

@@ -17,12 +17,12 @@
 package repository_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestRepository(t *testing.T) {

pkg/contract_watcher/shared/retriever/address_retriever.go

@@ -18,17 +18,17 @@ package retriever
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-// Address retriever is used to retrieve the addresses associated with a contract
+// AddressRetriever is used to retrieve the addresses associated with a contract
 type AddressRetriever interface {
 	RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error)
 }
@@ -38,14 +38,15 @@ type addressRetriever struct {
 	mode types.Mode
 }
 
-func NewAddressRetriever(db *postgres.DB, mode types.Mode) (r *addressRetriever) {
+// NewAddressRetriever returns a new AddressRetriever
+func NewAddressRetriever(db *postgres.DB, mode types.Mode) AddressRetriever {
 	return &addressRetriever{
 		db:   db,
 		mode: mode,
 	}
 }
 
-// Method to retrieve list of token-holding/contract-related addresses by iterating over available events
+// RetrieveTokenHolderAddresses is used to retrieve list of token-holding/contract-related addresses by iterating over available events
 // This generic method should work whether or not the argument/input names of the events meet the expected standard
 // This could be generalized to iterate over ALL events and pull out any address arguments
 func (r *addressRetriever) RetrieveTokenHolderAddresses(info contract.Contract) (map[common.Address]bool, error) {

pkg/contract_watcher/shared/retriever/address_retriever_test.go

@@ -48,12 +48,10 @@ var mockEvent = core.WatchedEvent{
 var _ = Describe("Address Retriever Test", func() {
 	var db *postgres.DB
 	var dataStore repository.EventRepository
-	var err error
 	var info *contract.Contract
 	var vulcanizeLogId int64
 	var log *types.Log
 	var r retriever.AddressRetriever
-	var addresses map[common.Address]bool
 	var wantedEvents = []string{"Transfer"}
 
 	BeforeEach(func() {
@@ -61,17 +59,18 @@ var _ = Describe("Address Retriever Test", func() {
 		mockEvent.LogID = vulcanizeLogId
 
 		event := info.Events["Transfer"]
-		err = info.GenerateFilters()
-		Expect(err).ToNot(HaveOccurred())
+		filterErr := info.GenerateFilters()
+		Expect(filterErr).ToNot(HaveOccurred())
 
 		c := converter.Converter{}
 		c.Update(info)
-		log, err = c.Convert(mockEvent, event)
-		Expect(err).ToNot(HaveOccurred())
+		var convertErr error
+		log, convertErr = c.Convert(mockEvent, event)
+		Expect(convertErr).ToNot(HaveOccurred())
 
 		dataStore = repository.NewEventRepository(db, types.FullSync)
-		dataStore.PersistLogs([]types.Log{*log}, event, info.Address, info.Name)
-		Expect(err).ToNot(HaveOccurred())
+		persistErr := dataStore.PersistLogs([]types.Log{*log}, event, info.Address, info.Name)
+		Expect(persistErr).ToNot(HaveOccurred())
 
 		r = retriever.NewAddressRetriever(db, types.FullSync)
 	})
@@ -82,8 +81,8 @@ var _ = Describe("Address Retriever Test", func() {
 
 	Describe("RetrieveTokenHolderAddresses", func() {
 		It("Retrieves a list of token holder addresses from persisted event logs", func() {
-			addresses, err = r.RetrieveTokenHolderAddresses(*info)
-			Expect(err).ToNot(HaveOccurred())
+			addresses, retrieveErr := r.RetrieveTokenHolderAddresses(*info)
+			Expect(retrieveErr).ToNot(HaveOccurred())
 
 			_, ok := addresses[common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")]
 			Expect(ok).To(Equal(true))
@@ -100,8 +99,8 @@ var _ = Describe("Address Retriever Test", func() {
 		})
 
 		It("Returns empty list when empty contract info is used", func() {
-			addresses, err = r.RetrieveTokenHolderAddresses(contract.Contract{})
-			Expect(err).ToNot(HaveOccurred())
+			addresses, retrieveErr := r.RetrieveTokenHolderAddresses(contract.Contract{})
+			Expect(retrieveErr).ToNot(HaveOccurred())
 			Expect(len(addresses)).To(Equal(0))
 		})
 	})