Address repo updates (#134 )

* Factor out get or create address into one sql string * Factor out getChecksumAddress method in address repo * Update address repo methods to not need a receiver * Move address repository to libraries/shared
Merge pull request #145 from vulcanize/contract-watcher-logging
2019-10-03 11:17:08 -05:00 · 2019-10-02 16:30:17 -05:00 · 2019-10-02 15:14:42 -05:00 · 2019-10-02 09:56:29 -05:00 · 2019-10-01 10:24:53 -05:00 · 2019-09-26 10:41:12 -05:00
61 changed files with 1475 additions and 713 deletions
@@ -17,6 +17,10 @@
 package cmd

 import (
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"os"
 	"plugin"
 	syn "sync"
@@ -25,9 +29,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"

-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
 	"github.com/vulcanize/vulcanizedb/utils"
@@ -179,12 +181,26 @@ func composeAndExecute() {
 	}

 	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
-		sw.AddTransformers(ethStorageInitializers)
-		wg.Add(1)
-		go watchEthStorage(&sw, &wg)
+		switch storageDiffsSource {
+		case "geth":
+			log.Debug("fetching storage diffs from geth pub sub")
+			rpcClient, _ := getClients()
+			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
+			payloadChan := make(chan statediff.Payload)
+			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		default:
+			log.Debug("fetching storage diffs from csv")
+			tailer := fs.FileTailer{Path: storageDiffsPath}
+			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		}
 	}

 	if len(ethContractInitializers) > 0 {
@@ -18,6 +18,10 @@ package cmd

 import (
 	"fmt"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"plugin"
 	syn "sync"
 	"time"
@@ -26,11 +30,9 @@ import (
 	"github.com/spf13/cobra"

 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"github.com/vulcanize/vulcanizedb/utils"
 )

@@ -123,12 +125,26 @@ func execute() {
 	}

 	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
-		sw.AddTransformers(ethStorageInitializers)
-		wg.Add(1)
-		go watchEthStorage(&sw, &wg)
+		switch storageDiffsSource {
+		case "geth":
+			log.Debug("fetching storage diffs from geth pub sub")
+			rpcClient, _ := getClients()
+			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
+			payloadChan := make(chan statediff.Payload)
+			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		default:
+			log.Debug("fetching storage diffs from csv")
+			tailer := fs.FileTailer{Path: storageDiffsPath}
+			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		}
 	}

 	if len(ethContractInitializers) > 0 {
@@ -166,7 +182,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	}
 }

-func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
+func watchEthStorage(w watcher.IStorageWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the StorageTransformerInitializer set using the storage watcher
 	LogWithCommand.Info("executing storage transformers")
@@ -174,8 +190,8 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 	defer ticker.Stop()
 	for range ticker.C {
 		errs := make(chan error)
-		rows := make(chan storageUtils.StorageDiffRow)
-		w.Execute(rows, errs, queueRecheckInterval)
+		diffs := make(chan storageUtils.StorageDiff)
+		w.Execute(diffs, errs, queueRecheckInterval)
 	}
 }

@@ -48,6 +48,7 @@ var (
 	recheckHeadersArg    bool
 	SubCommand           string
 	LogWithCommand       log.Entry
+	storageDiffsSource   string
 )

 const (
@@ -80,6 +81,7 @@ func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
 	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
+	storageDiffsSource = viper.GetString("storageDiffs.source")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -118,6 +120,7 @@ func init() {
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
+	rootCmd.PersistentFlags().String("storageDiffs-source", "csv", "where to get the state diffs: csv or geth")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")

@@ -129,6 +132,7 @@ func init() {
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
+	viper.BindPFlag("storageDiffs.source", rootCmd.PersistentFlags().Lookup("storageDiffs-source"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
@@ -152,6 +156,14 @@ func initConfig() {
 }

 func getBlockChain() *geth.BlockChain {
+	rpcClient, ethClient := getClients()
+	vdbEthClient := client.NewEthClient(ethClient)
+	vdbNode := node.MakeNode(rpcClient)
+	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
+	return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+}
+
+func getClients() (client.RpcClient, *ethclient.Client) {
 	rawRpcClient, err := rpc.Dial(ipc)

 	if err != nil {
@@ -159,8 +171,6 @@ func getBlockChain() *geth.BlockChain {
 	}
 	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
 	ethClient := ethclient.NewClient(rawRpcClient)
-	vdbEthClient := client.NewEthClient(ethClient)
-	vdbNode := node.MakeNode(rpcClient)
-	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+
+	return rpcClient, ethClient
 }
@@ -12,9 +12,11 @@ require (
 	github.com/ethereum/go-ethereum v1.9.5
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 // indirect
-	github.com/hashicorp/golang-lru v0.5.1
+	github.com/hashicorp/golang-lru v0.5.3
+	github.com/howeyc/fsnotify v0.9.0 // indirect
 	github.com/hpcloud/tail v1.0.0
 	github.com/huin/goupnp v1.0.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
@@ -42,8 +44,14 @@ require (
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.2 // indirect
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
+	golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 // indirect
 	golang.org/x/net v0.0.0-20190603091049-60506f45cf65
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/urfave/cli.v1 v1.0.0-00010101000000-000000000000 // indirect
 )
+
+replace github.com/ethereum/go-ethereum => github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101
+
+replace gopkg.in/urfave/cli.v1 => gopkg.in/urfave/cli.v1 v1.20.0
@@ -91,6 +91,8 @@ github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6/go.mod h1
 github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
@@ -241,6 +243,10 @@ github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2
 github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8=
 github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug=
 github.com/vulcanize/vulcanizedb v0.0.5/go.mod h1:utXkheCL9VjTfmuivuvRiAAyHh54GSN9XRQNEbFCA8k=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
@@ -252,6 +258,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 h1:4Yo9XtTfxfBCecLiBW8TYsFIdN7TkDhjGLWetFo4JSo=
+golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
@@ -281,6 +289,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aW
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -300,6 +310,7 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSv
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
 gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -18,7 +18,6 @@ package storage

 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
@@ -26,9 +25,9 @@ import (
 )

 type Transformer struct {
-	Address    common.Address
-	Mappings   storage.Mappings
-	Repository Repository
+	HashedAddress common.Hash
+	Mappings      storage.Mappings
+	Repository    Repository
 }

 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
@@ -37,18 +36,18 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.Stora
 	return transformer
 }

-func (transformer Transformer) ContractAddress() common.Address {
-	return transformer.Address
+func (transformer Transformer) KeccakContractAddress() common.Hash {
+	return transformer.HashedAddress
 }

-func (transformer Transformer) Execute(row utils.StorageDiffRow) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+func (transformer Transformer) Execute(diff utils.StorageDiff) error {
+	metadata, lookupErr := transformer.Mappings.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
-	value, decodeErr := utils.Decode(row, metadata)
+	value, decodeErr := utils.Decode(diff, metadata)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
 }
@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
@@ -38,21 +37,21 @@ var _ = Describe("Storage transformer", func() {
 		mappings = &mocks.MockMappings{}
 		repository = &mocks.MockStorageRepository{}
 		t = storage.Transformer{
-			Address:    common.Address{},
-			Mappings:   mappings,
-			Repository: repository,
+			HashedAddress: common.Hash{},
+			Mappings:      mappings,
+			Repository:    repository,
 		}
 	})

 	It("returns the contract address being watched", func() {
-		fakeAddress := common.HexToAddress("0x12345")
-		t.Address = fakeAddress
+		fakeAddress := utils.HexToKeccak256Hash("0x12345")
+		t.HashedAddress = fakeAddress

-		Expect(t.ContractAddress()).To(Equal(fakeAddress))
+		Expect(t.KeccakContractAddress()).To(Equal(fakeAddress))
 	})

 	It("looks up metadata for storage key", func() {
-		t.Execute(utils.StorageDiffRow{})
+		t.Execute(utils.StorageDiff{})

 		Expect(mappings.LookupCalled).To(BeTrue())
 	})
@@ -60,7 +59,7 @@ var _ = Describe("Storage transformer", func() {
 	It("returns error if lookup fails", func() {
 		mappings.LookupErr = fakes.FakeError

-		err := t.Execute(utils.StorageDiffRow{})
+		err := t.Execute(utils.StorageDiff{})

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -72,12 +71,12 @@ var _ = Describe("Storage transformer", func() {
 		rawValue := common.HexToAddress("0x12345")
 		fakeBlockNumber := 123
 		fakeBlockHash := "0x67890"
-		fakeRow := utils.StorageDiffRow{
-			Contract:     common.Address{},
-			BlockHash:    common.HexToHash(fakeBlockHash),
-			BlockHeight:  fakeBlockNumber,
-			StorageKey:   common.Hash{},
-			StorageValue: rawValue.Hash(),
+		fakeRow := utils.StorageDiff{
+			HashedAddress: common.Hash{},
+			BlockHash:     common.HexToHash(fakeBlockHash),
+			BlockHeight:   fakeBlockNumber,
+			StorageKey:    common.Hash{},
+			StorageValue:  rawValue.Hash(),
 		}

 		err := t.Execute(fakeRow)
@@ -95,7 +94,7 @@ var _ = Describe("Storage transformer", func() {
 		mappings.Metadata = fakeMetadata
 		repository.CreateErr = fakes.FakeError

-		err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+		err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -120,12 +119,12 @@ var _ = Describe("Storage transformer", func() {

 		It("passes the decoded data items to the repository", func() {
 			mappings.Metadata = fakeMetadata
-			fakeRow := utils.StorageDiffRow{
-				Contract:     common.Address{},
-				BlockHash:    common.HexToHash(fakeBlockHash),
-				BlockHeight:  fakeBlockNumber,
-				StorageKey:   common.Hash{},
-				StorageValue: rawValue.Hash(),
+			fakeRow := utils.StorageDiff{
+				HashedAddress: common.Hash{},
+				BlockHash:     common.HexToHash(fakeBlockHash),
+				BlockHeight:   fakeBlockNumber,
+				StorageKey:    common.Hash{},
+				StorageValue:  rawValue.Hash(),
 			}

 			err := t.Execute(fakeRow)
@@ -144,7 +143,7 @@ var _ = Describe("Storage transformer", func() {
 			mappings.Metadata = fakeMetadata
 			repository.CreateErr = fakes.FakeError

-			err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+			err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})

 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))
@@ -17,18 +17,12 @@
 package fetcher

 import (
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
+	"strings"
 )

-type IStorageFetcher interface {
-	FetchStorageDiffs(chan<- utils.StorageDiffRow, chan<- error)
-}
-
 type CsvTailStorageFetcher struct {
 	tailer fs.Tailer
 }
@@ -37,18 +31,18 @@ func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
 	return CsvTailStorageFetcher{tailer: tailer}
 }

-func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	t, tailErr := storageFetcher.tailer.Tail()
 	if tailErr != nil {
 		errs <- tailErr
 	}
-	log.Debug("fetching storage diffs...")
+	logrus.Debug("fetching storage diffs...")
 	for line := range t.Lines {
-		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+		diff, parseErr := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		if parseErr != nil {
 			errs <- parseErr
 		} else {
-			out <- row
+			out <- diff
 		}
 	}
 }
@@ -18,30 +18,28 @@ package fetcher_test

 import (
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/hpcloud/tail"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"strings"
+	"time"
 )

 var _ = Describe("Csv Tail Storage Fetcher", func() {
 	var (
 		errorsChannel  chan error
 		mockTailer     *fakes.MockTailer
-		rowsChannel    chan utils.StorageDiffRow
+		diffsChannel   chan utils.StorageDiff
 		storageFetcher fetcher.CsvTailStorageFetcher
 	)

 	BeforeEach(func() {
 		errorsChannel = make(chan error)
-		rowsChannel = make(chan utils.StorageDiffRow)
+		diffsChannel = make(chan utils.StorageDiff)
 		mockTailer = fakes.NewMockTailer()
 		storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
 	})
@@ -49,7 +47,7 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 	It("adds error to errors channel if tailing file fails", func(done Done) {
 		mockTailer.TailErr = fakes.FakeError

-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)

 		Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
 		close(done)
@@ -58,24 +56,24 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 	It("adds parsed csv row to rows channel for storage diff", func(done Done) {
 		line := getFakeLine()

-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line

-		expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
+		expectedRow, err := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		Expect(err).NotTo(HaveOccurred())
-		Expect(<-rowsChannel).To(Equal(expectedRow))
+		Expect(<-diffsChannel).To(Equal(expectedRow))
 		close(done)
 	})

 	It("adds error to errors channel if parsing csv fails", func(done Done) {
 		line := &tail.Line{Text: "invalid"}

-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line

 		Expect(<-errorsChannel).To(HaveOccurred())
 		select {
-		case <-rowsChannel:
+		case <-diffsChannel:
 			Fail("value passed to rows channel on error")
 		default:
 			Succeed()
@@ -0,0 +1,81 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher
+
+import (
+	"fmt"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+)
+
+type GethRpcStorageFetcher struct {
+	statediffPayloadChan chan statediff.Payload
+	streamer             streamer.Streamer
+}
+
+func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan statediff.Payload) GethRpcStorageFetcher {
+	return GethRpcStorageFetcher{
+		statediffPayloadChan: statediffPayloadChan,
+		streamer:             streamer,
+	}
+}
+
+func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
+	ethStatediffPayloadChan := fetcher.statediffPayloadChan
+	clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
+	if clientSubErr != nil {
+		errs <- clientSubErr
+		panic(fmt.Sprintf("Error creating a geth client subscription: %v", clientSubErr))
+	}
+	logrus.Info("Successfully created a geth client subscription: ", clientSubscription)
+
+	for {
+		diff := <-ethStatediffPayloadChan
+		logrus.Trace("received a statediff")
+		stateDiff := new(statediff.StateDiff)
+		decodeErr := rlp.DecodeBytes(diff.StateDiffRlp, stateDiff)
+		if decodeErr != nil {
+			logrus.Warn("Error decoding state diff into RLP: ", decodeErr)
+			errs <- decodeErr
+		}
+
+		accounts := getAccountsFromDiff(*stateDiff)
+		logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
+		for _, account := range accounts {
+			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
+			for _, storage := range account.Storage {
+				diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
+				logrus.Trace("adding storage diff to out channel",
+					"keccak of address: ", diff.HashedAddress.Hex(),
+					"block height: ", diff.BlockHeight,
+					"storage key: ", diff.StorageKey.Hex(),
+					"storage value: ", diff.StorageValue.Hex())
+				if formatErr != nil {
+					errs <- formatErr
+				}
+
+				out <- diff
+			}
+		}
+	}
+}
+
+func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
+	accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
+	return append(accounts, stateDiff.DeletedAccounts...)
+}
@@ -0,0 +1,172 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+type MockStoragediffStreamer struct {
+	subscribeError    error
+	PassedPayloadChan chan statediff.Payload
+	streamPayloads    []statediff.Payload
+}
+
+func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	clientSubscription := rpc.ClientSubscription{}
+	streamer.PassedPayloadChan = statediffPayloadChan
+
+	go func() {
+		for _, payload := range streamer.streamPayloads {
+			streamer.PassedPayloadChan <- payload
+		}
+	}()
+
+	return &clientSubscription, streamer.subscribeError
+}
+
+func (streamer *MockStoragediffStreamer) SetSubscribeError(err error) {
+	streamer.subscribeError = err
+}
+
+func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payload) {
+	streamer.streamPayloads = payloads
+}
+
+var _ = Describe("Geth RPC Storage Fetcher", func() {
+	var streamer MockStoragediffStreamer
+	var statediffPayloadChan chan statediff.Payload
+	var statediffFetcher fetcher.GethRpcStorageFetcher
+	var storagediffChan chan utils.StorageDiff
+	var errorChan chan error
+
+	BeforeEach(func() {
+		streamer = MockStoragediffStreamer{}
+		statediffPayloadChan = make(chan statediff.Payload, 1)
+		statediffFetcher = fetcher.NewGethRpcStorageFetcher(&streamer, statediffPayloadChan)
+		storagediffChan = make(chan utils.StorageDiff)
+		errorChan = make(chan error)
+	})
+
+	It("adds errors to error channel if the RPC subscription fails and panics", func(done Done) {
+		streamer.SetSubscribeError(fakes.FakeError)
+
+		go func() {
+			failedSub := func() {
+				statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+			}
+			Expect(failedSub).To(Panic())
+		}()
+
+		Expect(<-errorChan).To(MatchError(fakes.FakeError))
+		close(done)
+	})
+
+	It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		streamedPayload := <-statediffPayloadChan
+		Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload))
+		Expect(streamer.PassedPayloadChan).To(Equal(statediffPayloadChan))
+		close(done)
+	})
+
+	It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) {
+		badStatediffPayload := statediff.Payload{}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("EOF"))
+
+		close(done)
+	})
+
+	It("adds parsed statediff payloads to the rows channel", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		height := test_data.BlockNumber
+		intHeight := int(height.Int64())
+		createdExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
+		}
+		updatedExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.LargeStorageValue),
+		}
+		deletedExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
+		}
+
+		createdStateDiff := <-storagediffChan
+		updatedStateDiff := <-storagediffChan
+		deletedStateDiff := <-storagediffChan
+
+		Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff))
+		Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff))
+		Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff))
+
+		close(done)
+	})
+
+	It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) {
+		accountDiffs := test_data.CreatedAccountDiffs
+		accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue}
+
+		stateDiff := statediff.StateDiff{
+			BlockNumber:     test_data.BlockNumber,
+			BlockHash:       common.HexToHash(test_data.BlockHash),
+			CreatedAccounts: accountDiffs,
+		}
+
+		stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+		Expect(err).NotTo(HaveOccurred())
+
+		badStatediffPayload := statediff.Payload{
+			StateDiffRlp: stateDiffRlp,
+		}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("rlp: input contains more than one value"))
+
+		close(done)
+	})
+})
@@ -0,0 +1,21 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher
+
+import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+
+type IStorageFetcher interface {
+	FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error)
+}
@@ -19,21 +19,21 @@ package mocks
 import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"

 type MockStorageFetcher struct {
-	RowsToReturn []utils.StorageDiffRow
-	ErrsToReturn []error
+	DiffsToReturn []utils.StorageDiff
+	ErrsToReturn  []error
 }

 func NewMockStorageFetcher() *MockStorageFetcher {
 	return &MockStorageFetcher{}
 }

-func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	defer close(out)
 	defer close(errs)
 	for _, err := range fetcher.ErrsToReturn {
 		errs <- err
 	}
-	for _, row := range fetcher.RowsToReturn {
-		out <- row
+	for _, diff := range fetcher.DiffsToReturn {
+		out <- diff
 	}
 }
@@ -23,16 +23,16 @@ import (
 type MockStorageQueue struct {
 	AddCalled      bool
 	AddError       error
-	AddPassedRow   utils.StorageDiffRow
+	AddPassedDiff  utils.StorageDiff
 	DeleteErr      error
 	DeletePassedId int
 	GetAllErr      error
-	RowsToReturn   []utils.StorageDiffRow
+	DiffsToReturn  []utils.StorageDiff
 }

-func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error {
+func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
 	queue.AddCalled = true
-	queue.AddPassedRow = row
+	queue.AddPassedDiff = diff
 	return queue.AddError
 }

@@ -41,6 +41,6 @@ func (queue *MockStorageQueue) Delete(id int) error {
 	return queue.DeleteErr
 }

-func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
-	return queue.RowsToReturn, queue.GetAllErr
+func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
+	return queue.DiffsToReturn, queue.GetAllErr
 }
@@ -18,25 +18,24 @@ package mocks

 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )

 type MockStorageTransformer struct {
-	Address    common.Address
-	ExecuteErr error
-	PassedRow  utils.StorageDiffRow
+	KeccakOfAddress common.Hash
+	ExecuteErr      error
+	PassedDiff      utils.StorageDiff
 }

-func (transformer *MockStorageTransformer) Execute(row utils.StorageDiffRow) error {
-	transformer.PassedRow = row
+func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
+	transformer.PassedDiff = diff
 	return transformer.ExecuteErr
 }

-func (transformer *MockStorageTransformer) ContractAddress() common.Address {
-	return transformer.Address
+func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash {
+	return transformer.KeccakOfAddress
 }

 func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {
@@ -0,0 +1,73 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/jmoiron/sqlx"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+const getOrCreateAddressQuery = `WITH addressId AS (
+			INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id
+		)
+		SELECT id FROM addresses WHERE address = $1
+		UNION
+		SELECT id FROM addressId`
+
+func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
+	checksumAddress := getChecksumAddress(address)
+
+	var addressId int64
+	getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
+
+	return addressId, getOrCreateErr
+}
+
+func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
+	checksumAddress := getChecksumAddress(address)
+
+	var addressId int64
+	getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
+
+	return addressId, getOrCreateErr
+}
+
+func GetAddressById(db *postgres.DB, id int64) (string, error) {
+	var address string
+	getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
+	return address, getErr
+}
+
+func getChecksumAddress(address string) string {
+	stringAddressToCommonAddress := common.HexToAddress(address)
+	return stringAddressToCommonAddress.Hex()
+}
@@ -1,22 +1,23 @@
 // VulcanizeDB
 // Copyright © 2019 Vulcanize
-
+//
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
-
+//
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
-
+//
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.

-package repositories_test
+package repository_test

 import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"strings"

 	"github.com/jmoiron/sqlx"
@@ -24,7 +25,6 @@ import (
 	. "github.com/onsi/gomega"

 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
@@ -32,13 +32,11 @@ import (
 var _ = Describe("address lookup", func() {
 	var (
 		db      *postgres.DB
-		repo    repositories.AddressRepository
 		address = fakes.FakeAddress.Hex()
 	)
 	BeforeEach(func() {
 		db = test_config.NewTestDB(test_config.NewTestNode())
 		test_config.CleanTestDB(db)
-		repo = repositories.AddressRepository{}
 	})

 	AfterEach(func() {
@@ -52,7 +50,7 @@ var _ = Describe("address lookup", func() {

 	Describe("GetOrCreateAddress", func() {
 		It("creates an address record", func() {
-			addressId, createErr := repo.GetOrCreateAddress(db, address)
+			addressId, createErr := repository.GetOrCreateAddress(db, address)
 			Expect(createErr).NotTo(HaveOccurred())

 			var actualAddress dbAddress
@@ -63,10 +61,10 @@ var _ = Describe("address lookup", func() {
 		})

 		It("returns the existing record id if the address already exists", func() {
-			createId, createErr := repo.GetOrCreateAddress(db, address)
+			createId, createErr := repository.GetOrCreateAddress(db, address)
 			Expect(createErr).NotTo(HaveOccurred())

-			getId, getErr := repo.GetOrCreateAddress(db, address)
+			getId, getErr := repository.GetOrCreateAddress(db, address)
 			Expect(getErr).NotTo(HaveOccurred())

 			var addressCount int
@@ -78,20 +76,20 @@ var _ = Describe("address lookup", func() {

 		It("gets upper-cased addresses", func() {
 			upperAddress := strings.ToUpper(address)
-			upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddress(db, upperAddress)
 			Expect(createErr).NotTo(HaveOccurred())

-			mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
 		})

 		It("gets lower-cased addresses", func() {
 			lowerAddress := strings.ToLower(address)
-			upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddress(db, lowerAddress)
 			Expect(createErr).NotTo(HaveOccurred())

-			mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
 		})
@@ -112,7 +110,7 @@ var _ = Describe("address lookup", func() {
 		})

 		It("creates an address record", func() {
-			addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			addressId, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(createErr).NotTo(HaveOccurred())
 			commitErr := tx.Commit()
 			Expect(commitErr).NotTo(HaveOccurred())
@@ -125,10 +123,10 @@ var _ = Describe("address lookup", func() {
 		})

 		It("returns the existing record id if the address already exists", func() {
-			_, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			_, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(createErr).NotTo(HaveOccurred())

-			_, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			_, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			tx.Commit()

@@ -139,10 +137,10 @@ var _ = Describe("address lookup", func() {

 		It("gets upper-cased addresses", func() {
 			upperAddress := strings.ToUpper(address)
-			upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, upperAddress)
 			Expect(createErr).NotTo(HaveOccurred())

-			mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			tx.Commit()

@@ -151,10 +149,10 @@ var _ = Describe("address lookup", func() {

 		It("gets lower-cased addresses", func() {
 			lowerAddress := strings.ToLower(address)
-			upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress)
+			upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, lowerAddress)
 			Expect(createErr).NotTo(HaveOccurred())

-			mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
 			Expect(getErr).NotTo(HaveOccurred())
 			tx.Commit()

@@ -164,16 +162,16 @@ var _ = Describe("address lookup", func() {

 	Describe("GetAddressById", func() {
 		It("gets and address by it's id", func() {
-			addressId, createErr := repo.GetOrCreateAddress(db, address)
+			addressId, createErr := repository.GetOrCreateAddress(db, address)
 			Expect(createErr).NotTo(HaveOccurred())

-			actualAddress, getErr := repo.GetAddressById(db, addressId)
+			actualAddress, getErr := repository.GetAddressById(db, addressId)
 			Expect(getErr).NotTo(HaveOccurred())
 			Expect(actualAddress).To(Equal(address))
 		})

 		It("returns an error if the id doesn't exist", func() {
-			_, getErr := repo.GetAddressById(db, 0)
+			_, getErr := repository.GetAddressById(db, 0)
 			Expect(getErr).To(HaveOccurred())
 			Expect(getErr).To(MatchError("sql: no rows in result set"))
 		})
@@ -46,16 +46,29 @@ const (
 	IndexEleven = "000000000000000000000000000000000000000000000000000000000000000b"
 )

+func AddHashedKeys(currentMappings map[common.Hash]utils.StorageValueMetadata) map[common.Hash]utils.StorageValueMetadata {
+	copyOfCurrentMappings := make(map[common.Hash]utils.StorageValueMetadata)
+	for k, v := range currentMappings {
+		copyOfCurrentMappings[k] = v
+	}
+	for k, v := range copyOfCurrentMappings {
+		currentMappings[hashKey(k)] = v
+	}
+	return currentMappings
+}
+
+func hashKey(key common.Hash) common.Hash {
+	return crypto.Keccak256Hash(key.Bytes())
+}
+
 func GetMapping(indexOnContract, key string) common.Hash {
 	keyBytes := common.FromHex(key + indexOnContract)
-	encoded := crypto.Keccak256(keyBytes)
-	return common.BytesToHash(encoded)
+	return crypto.Keccak256Hash(keyBytes)
 }

 func GetNestedMapping(indexOnContract, primaryKey, secondaryKey string) common.Hash {
 	primaryMappingIndex := crypto.Keccak256(common.FromHex(primaryKey + indexOnContract))
-	secondaryMappingIndex := crypto.Keccak256(common.FromHex(secondaryKey), primaryMappingIndex)
-	return common.BytesToHash(secondaryMappingIndex)
+	return crypto.Keccak256Hash(common.FromHex(secondaryKey), primaryMappingIndex)
 }

 func GetIncrementedKey(original common.Hash, incrementBy int64) common.Hash {
@@ -5,9 +5,31 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 )

 var _ = Describe("Mappings", func() {
+	Describe("AddHashedKeys", func() {
+		It("returns a copy of the map with an additional slot for the hashed version of every key", func() {
+			fakeMap := map[common.Hash]utils.StorageValueMetadata{}
+			fakeStorageKey := common.HexToHash("72c72de6b203d67cb6cd54fc93300109fcc6fd6eac88e390271a3d548794d800")
+			var fakeMappingKey utils.Key = "fakeKey"
+			fakeMetadata := utils.StorageValueMetadata{
+				Name: "fakeName",
+				Keys: map[utils.Key]string{fakeMappingKey: "fakeValue"},
+				Type: utils.Uint48,
+			}
+			fakeMap[fakeStorageKey] = fakeMetadata
+
+			result := storage.AddHashedKeys(fakeMap)
+
+			Expect(len(result)).To(Equal(2))
+			expectedHashedStorageKey := common.HexToHash("2165edb4e1c37b99b60fa510d84f939dd35d5cd1d1c8f299d6456ea09df65a76")
+			Expect(fakeMap[fakeStorageKey]).To(Equal(fakeMetadata))
+			Expect(fakeMap[expectedHashedStorageKey]).To(Equal(fakeMetadata))
+		})
+	})
+
 	Describe("GetMapping", func() {
 		It("returns the storage key for a mapping when passed the mapping's index on the contract and the desired value's key", func() {
 			// ex. solidity:
@@ -22,9 +22,9 @@ import (
 )

 type IStorageQueue interface {
-	Add(row utils.StorageDiffRow) error
+	Add(diff utils.StorageDiff) error
 	Delete(id int) error
-	GetAll() ([]utils.StorageDiffRow, error)
+	GetAll() ([]utils.StorageDiff, error)
 }

 type StorageQueue struct {
@@ -35,11 +35,11 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
 	return StorageQueue{db: db}
 }

-func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
+func (queue StorageQueue) Add(diff utils.StorageDiff) error {
 	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
 		block_hash, block_height, storage_key, storage_value) VALUES
-		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, row.Contract.Bytes(), row.BlockHash.Bytes(),
-		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
+		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.HashedAddress.Bytes(), diff.BlockHash.Bytes(),
+		diff.BlockHeight, diff.StorageKey.Bytes(), diff.StorageValue.Bytes())
 	return err
 }

@@ -48,8 +48,8 @@ func (queue StorageQueue) Delete(id int) error {
 	return err
 }

-func (queue StorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
-	var result []utils.StorageDiffRow
+func (queue StorageQueue) GetAll() ([]utils.StorageDiff, error) {
+	var result []utils.StorageDiff
 	err := queue.db.Select(&result, `SELECT * FROM public.queued_storage`)
 	return result, err
 }
@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -30,35 +29,36 @@ import (
 var _ = Describe("Storage queue", func() {
 	var (
 		db    *postgres.DB
-		row   utils.StorageDiffRow
+		diff  utils.StorageDiff
 		queue storage.IStorageQueue
 	)

 	BeforeEach(func() {
-		row = utils.StorageDiffRow{
-			Contract:     common.HexToAddress("0x123456"),
-			BlockHash:    common.HexToHash("0x678901"),
-			BlockHeight:  987,
-			StorageKey:   common.HexToHash("0x654321"),
-			StorageValue: common.HexToHash("0x198765"),
+		fakeAddr := "0x123456"
+		diff = utils.StorageDiff{
+			HashedAddress: utils.HexToKeccak256Hash(fakeAddr),
+			BlockHash:     common.HexToHash("0x678901"),
+			BlockHeight:   987,
+			StorageKey:    common.HexToHash("0x654321"),
+			StorageValue:  common.HexToHash("0x198765"),
 		}
 		db = test_config.NewTestDB(test_config.NewTestNode())
 		test_config.CleanTestDB(db)
 		queue = storage.NewStorageQueue(db)
-		addErr := queue.Add(row)
+		addErr := queue.Add(diff)
 		Expect(addErr).NotTo(HaveOccurred())
 	})

 	Describe("Add", func() {
-		It("adds a storage row to the db", func() {
-			var result utils.StorageDiffRow
+		It("adds a storage diff to the db", func() {
+			var result utils.StorageDiff
 			getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
 			Expect(getErr).NotTo(HaveOccurred())
-			Expect(result).To(Equal(row))
+			Expect(result).To(Equal(diff))
 		})

-		It("does not duplicate storage rows", func() {
-			addErr := queue.Add(row)
+		It("does not duplicate storage diffs", func() {
+			addErr := queue.Add(diff)
 			Expect(addErr).NotTo(HaveOccurred())
 			var count int
 			getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`)
@@ -67,12 +67,12 @@ var _ = Describe("Storage queue", func() {
 		})
 	})

-	It("deletes storage row from db", func() {
-		rows, getErr := queue.GetAll()
+	It("deletes storage diff from db", func() {
+		diffs, getErr := queue.GetAll()
 		Expect(getErr).NotTo(HaveOccurred())
-		Expect(len(rows)).To(Equal(1))
+		Expect(len(diffs)).To(Equal(1))

-		err := queue.Delete(rows[0].Id)
+		err := queue.Delete(diffs[0].Id)

 		Expect(err).NotTo(HaveOccurred())
 		remainingRows, secondGetErr := queue.GetAll()
@@ -80,33 +80,34 @@ var _ = Describe("Storage queue", func() {
 		Expect(len(remainingRows)).To(BeZero())
 	})

-	It("gets all storage rows from db", func() {
-		rowTwo := utils.StorageDiffRow{
-			Contract:     common.HexToAddress("0x123456"),
-			BlockHash:    common.HexToHash("0x678902"),
-			BlockHeight:  988,
-			StorageKey:   common.HexToHash("0x654322"),
-			StorageValue: common.HexToHash("0x198766"),
+	It("gets all storage diffs from db", func() {
+		fakeAddr := "0x234567"
+		diffTwo := utils.StorageDiff{
+			HashedAddress: utils.HexToKeccak256Hash(fakeAddr),
+			BlockHash:     common.HexToHash("0x678902"),
+			BlockHeight:   988,
+			StorageKey:    common.HexToHash("0x654322"),
+			StorageValue:  common.HexToHash("0x198766"),
 		}
-		addErr := queue.Add(rowTwo)
+		addErr := queue.Add(diffTwo)
 		Expect(addErr).NotTo(HaveOccurred())

-		rows, err := queue.GetAll()
+		diffs, err := queue.GetAll()

 		Expect(err).NotTo(HaveOccurred())
-		Expect(len(rows)).To(Equal(2))
-		Expect(rows[0]).NotTo(Equal(rows[1]))
-		Expect(rows[0].Id).NotTo(BeZero())
-		Expect(rows[0].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
-		Expect(rows[0].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
-		Expect(rows[0].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
-		Expect(rows[0].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
-		Expect(rows[0].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
-		Expect(rows[1].Id).NotTo(BeZero())
-		Expect(rows[1].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
-		Expect(rows[1].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
-		Expect(rows[1].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
-		Expect(rows[1].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
-		Expect(rows[1].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
+		Expect(len(diffs)).To(Equal(2))
+		Expect(diffs[0]).NotTo(Equal(diffs[1]))
+		Expect(diffs[0].Id).NotTo(BeZero())
+		Expect(diffs[0].HashedAddress).To(Or(Equal(diff.HashedAddress), Equal(diffTwo.HashedAddress)))
+		Expect(diffs[0].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
+		Expect(diffs[0].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
+		Expect(diffs[0].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
+		Expect(diffs[0].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
+		Expect(diffs[1].Id).NotTo(BeZero())
+		Expect(diffs[1].HashedAddress).To(Or(Equal(diff.HashedAddress), Equal(diffTwo.HashedAddress)))
+		Expect(diffs[1].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
+		Expect(diffs[1].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
+		Expect(diffs[1].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
+		Expect(diffs[1].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
 	})
 })
@@ -27,20 +27,20 @@ const (
 	bitsPerByte = 8
 )

-func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
+func Decode(diff StorageDiff, metadata StorageValueMetadata) (interface{}, error) {
 	switch metadata.Type {
 	case Uint256:
-		return decodeInteger(row.StorageValue.Bytes()), nil
+		return decodeInteger(diff.StorageValue.Bytes()), nil
 	case Uint48:
-		return decodeInteger(row.StorageValue.Bytes()), nil
+		return decodeInteger(diff.StorageValue.Bytes()), nil
 	case Uint128:
-		return decodeInteger(row.StorageValue.Bytes()), nil
+		return decodeInteger(diff.StorageValue.Bytes()), nil
 	case Address:
-		return decodeAddress(row.StorageValue.Bytes()), nil
+		return decodeAddress(diff.StorageValue.Bytes()), nil
 	case Bytes32:
-		return row.StorageValue.Hex(), nil
+		return diff.StorageValue.Hex(), nil
 	case PackedSlot:
-		return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
+		return decodePackedSlot(diff.StorageValue.Bytes(), metadata.PackedTypes), nil
 	default:
 		panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
 	}
@@ -29,10 +29,10 @@ import (
 var _ = Describe("Storage decoder", func() {
 	It("decodes uint256", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
-		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		diff := utils.StorageDiff{StorageValue: fakeInt}
 		metadata := utils.StorageValueMetadata{Type: utils.Uint256}

-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@@ -40,10 +40,10 @@ var _ = Describe("Storage decoder", func() {

 	It("decodes uint128", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
-		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		diff := utils.StorageDiff{StorageValue: fakeInt}
 		metadata := utils.StorageValueMetadata{Type: utils.Uint128}

-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@@ -51,10 +51,10 @@ var _ = Describe("Storage decoder", func() {

 	It("decodes uint48", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
-		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		diff := utils.StorageDiff{StorageValue: fakeInt}
 		metadata := utils.StorageValueMetadata{Type: utils.Uint48}

-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
@@ -62,10 +62,10 @@ var _ = Describe("Storage decoder", func() {

 	It("decodes address", func() {
 		fakeAddress := common.HexToAddress("0x12345")
-		row := utils.StorageDiffRow{StorageValue: fakeAddress.Hash()}
+		diff := utils.StorageDiff{StorageValue: fakeAddress.Hash()}
 		metadata := utils.StorageValueMetadata{Type: utils.Address}

-		result, err := utils.Decode(row, metadata)
+		result, err := utils.Decode(diff, metadata)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(fakeAddress.Hex()))
@@ -75,7 +75,7 @@ var _ = Describe("Storage decoder", func() {
 		It("decodes uint48 items", func() {
 			//this is a real storage data example
 			packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			diff := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Uint48
 			packedTypes[1] = utils.Uint48
@@ -85,7 +85,7 @@ var _ = Describe("Storage decoder", func() {
 				PackedTypes: packedTypes,
 			}

-			result, err := utils.Decode(row, metadata)
+			result, err := utils.Decode(diff, metadata)
 			decodedValues := result.(map[int]string)

 			Expect(err).NotTo(HaveOccurred())
@@ -99,7 +99,7 @@ var _ = Describe("Storage decoder", func() {
 			packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"

 			packedStorage := common.HexToHash(packedStorageHex)
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			diff := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Uint48
 			packedTypes[1] = utils.Uint48
@@ -112,7 +112,7 @@ var _ = Describe("Storage decoder", func() {
 				PackedTypes: packedTypes,
 			}

-			result, err := utils.Decode(row, metadata)
+			result, err := utils.Decode(diff, metadata)
 			decodedValues := result.(map[int]string)

 			Expect(err).NotTo(HaveOccurred())
@@ -129,7 +129,7 @@ var _ = Describe("Storage decoder", func() {
 			packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
 				"0000000000000000AB54A98CEB1F0AD2"
 			packedStorage := common.HexToHash(packedStorageHex)
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			diff := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Uint128
 			packedTypes[1] = utils.Uint128
@@ -139,7 +139,7 @@ var _ = Describe("Storage decoder", func() {
 				PackedTypes: packedTypes,
 			}

-			result, err := utils.Decode(row, metadata)
+			result, err := utils.Decode(diff, metadata)
 			decodedValues := result.(map[int]string)

 			Expect(err).NotTo(HaveOccurred())
@@ -151,7 +151,7 @@ var _ = Describe("Storage decoder", func() {
 			//TODO: replace with real data when available
 			addressHex := "0000000000000000000000000000000000012345"
 			packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
-			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			row := utils.StorageDiff{StorageValue: packedStorage}
 			packedTypes := map[int]utils.ValueType{}
 			packedTypes[0] = utils.Address
 			packedTypes[1] = utils.Uint48
@@ -0,0 +1,73 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"strconv"
+)
+
+const ExpectedRowLength = 5
+
+type StorageDiff struct {
+	Id            int
+	HashedAddress common.Hash `db:"contract"`
+	BlockHash     common.Hash `db:"block_hash"`
+	BlockHeight   int         `db:"block_height"`
+	StorageKey    common.Hash `db:"storage_key"`
+	StorageValue  common.Hash `db:"storage_value"`
+}
+
+func FromParityCsvRow(csvRow []string) (StorageDiff, error) {
+	if len(csvRow) != ExpectedRowLength {
+		return StorageDiff{}, ErrRowMalformed{Length: len(csvRow)}
+	}
+	height, err := strconv.Atoi(csvRow[2])
+	if err != nil {
+		return StorageDiff{}, err
+	}
+	return StorageDiff{
+		HashedAddress: HexToKeccak256Hash(csvRow[0]),
+		BlockHash:     common.HexToHash(csvRow[1]),
+		BlockHeight:   height,
+		StorageKey:    common.HexToHash(csvRow[3]),
+		StorageValue:  common.HexToHash(csvRow[4]),
+	}, nil
+}
+
+func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiff, error) {
+	var decodedValue []byte
+	err := rlp.DecodeBytes(storage.Value, &decodedValue)
+	if err != nil {
+		return StorageDiff{}, err
+	}
+
+	return StorageDiff{
+		HashedAddress: common.BytesToHash(account.Key),
+		BlockHash:     stateDiff.BlockHash,
+		BlockHeight:   int(stateDiff.BlockNumber.Int64()),
+		StorageKey:    common.BytesToHash(storage.Key),
+		StorageValue:  common.BytesToHash(decodedValue),
+	}, nil
+}
+
+func HexToKeccak256Hash(hex string) common.Hash {
+	return crypto.Keccak256Hash(common.FromHex(hex))
+}
@@ -0,0 +1,121 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package utils_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"math/big"
+	"math/rand"
+)
+
+var _ = Describe("Storage row parsing", func() {
+	Describe("FromParityCsvRow", func() {
+		It("converts an array of strings to a row struct", func() {
+			contract := "0x123"
+			blockHash := "0x456"
+			blockHeight := "789"
+			storageKey := "0x987"
+			storageValue := "0x654"
+			data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
+
+			result, err := utils.FromParityCsvRow(data)
+
+			Expect(err).NotTo(HaveOccurred())
+			expectedKeccakOfContractAddress := utils.HexToKeccak256Hash(contract)
+			Expect(result.HashedAddress).To(Equal(expectedKeccakOfContractAddress))
+			Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
+			Expect(result.BlockHeight).To(Equal(789))
+			Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
+			Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
+		})
+
+		It("returns an error if row is missing data", func() {
+			_, err := utils.FromParityCsvRow([]string{"0x123"})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
+		})
+
+		It("returns error if block height malformed", func() {
+			_, err := utils.FromParityCsvRow([]string{"", "", "", "", ""})
+
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("FromGethStateDiff", func() {
+		var (
+			accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
+			stateDiff   = &statediff.StateDiff{
+				BlockNumber: big.NewInt(rand.Int63()),
+				BlockHash:   fakes.FakeHash,
+			}
+		)
+
+		It("adds relevant fields to diff", func() {
+			storageValueBytes := []byte{3}
+			storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
+			Expect(encodeErr).NotTo(HaveOccurred())
+
+			storageDiff := statediff.StorageDiff{
+				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
+				Value: storageValueRlp,
+			}
+
+			result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).NotTo(HaveOccurred())
+
+			expectedAddress := common.BytesToHash(accountDiff.Key)
+			Expect(result.HashedAddress).To(Equal(expectedAddress))
+			Expect(result.BlockHash).To(Equal(fakes.FakeHash))
+			expectedBlockHeight := int(stateDiff.BlockNumber.Int64())
+			Expect(result.BlockHeight).To(Equal(expectedBlockHeight))
+			expectedStorageKey := common.BytesToHash(storageDiff.Key)
+			Expect(result.StorageKey).To(Equal(expectedStorageKey))
+			expectedStorageValue := common.BytesToHash(storageValueBytes)
+			Expect(result.StorageValue).To(Equal(expectedStorageValue))
+		})
+
+		It("handles decoding large storage values from their RLP", func() {
+			storageValueBytes := []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6}
+			storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
+			Expect(encodeErr).NotTo(HaveOccurred())
+
+			storageDiff := statediff.StorageDiff{
+				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
+				Value: storageValueRlp,
+			}
+
+			result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes)))
+		})
+
+		It("returns an err if decoding the storage value Rlp fails", func() {
+			_, err := utils.FromGethStateDiff(accountDiff, stateDiff, test_data.StorageWithBadValue)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("rlp: input contains more than one value"))
+		})
+	})
+})
@@ -1,51 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package utils
-
-import (
-	"strconv"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-const ExpectedRowLength = 5
-
-type StorageDiffRow struct {
-	Id           int
-	Contract     common.Address
-	BlockHash    common.Hash `db:"block_hash"`
-	BlockHeight  int         `db:"block_height"`
-	StorageKey   common.Hash `db:"storage_key"`
-	StorageValue common.Hash `db:"storage_value"`
-}
-
-func FromStrings(csvRow []string) (StorageDiffRow, error) {
-	if len(csvRow) != ExpectedRowLength {
-		return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)}
-	}
-	height, err := strconv.Atoi(csvRow[2])
-	if err != nil {
-		return StorageDiffRow{}, err
-	}
-	return StorageDiffRow{
-		Contract:     common.HexToAddress(csvRow[0]),
-		BlockHash:    common.HexToHash(csvRow[1]),
-		BlockHeight:  height,
-		StorageKey:   common.HexToHash(csvRow[3]),
-		StorageValue: common.HexToHash(csvRow[4]),
-	}, nil
-}
@@ -1,58 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package utils_test
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-)
-
-var _ = Describe("Storage row parsing", func() {
-	It("converts an array of strings to a row struct", func() {
-		contract := "0x123"
-		blockHash := "0x456"
-		blockHeight := "789"
-		storageKey := "0x987"
-		storageValue := "0x654"
-		data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
-
-		result, err := utils.FromStrings(data)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
-		Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
-		Expect(result.BlockHeight).To(Equal(789))
-		Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
-		Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
-	})
-
-	It("returns an error if row is missing data", func() {
-		_, err := utils.FromStrings([]string{"0x123"})
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
-	})
-
-	It("returns error if block height malformed", func() {
-		_, err := utils.FromStrings([]string{"", "", "", "", ""})
-
-		Expect(err).To(HaveOccurred())
-	})
-})
@@ -0,0 +1,41 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package streamer
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+type Streamer interface {
+	Stream(chan statediff.Payload) (*rpc.ClientSubscription, error)
+}
+
+type StateDiffStreamer struct {
+	client core.RpcClient
+}
+
+func (streamer *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	logrus.Info("streaming diffs from geth")
+	return streamer.client.Subscribe("statediff", payloadChan, "stream")
+}
+
+func NewStateDiffStreamer(client core.RpcClient) StateDiffStreamer {
+	return StateDiffStreamer{
+		client: client,
+	}
+}
@@ -0,0 +1,35 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package streamer_test
+
+import (
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("StateDiff Streamer", func() {
+	It("subscribes to the geth statediff service", func() {
+		client := &fakes.MockRpcClient{}
+		streamer := streamer.NewStateDiffStreamer(client)
+		payloadChan := make(chan statediff.Payload)
+		_, err := streamer.Stream(payloadChan)
+		Expect(err).NotTo(HaveOccurred())
+
+		client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"})
+	})
+})
@@ -0,0 +1,19 @@
+package streamer_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestStreamer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Streamer Suite")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})
@@ -0,0 +1,137 @@
+// Copyright 2018 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test_data
+
+import (
+	"errors"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"math/big"
+	"math/rand"
+)
+
+var (
+	BlockNumber             = big.NewInt(rand.Int63())
+	BlockHash               = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
+	CodeHash                = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	NewNonceValue           = rand.Uint64()
+	NewBalanceValue         = rand.Int63()
+	ContractRoot            = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+	StoragePath             = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+	StorageKey              = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+	SmallStorageValue       = common.Hex2Bytes("03")
+	SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue)
+	storageWithSmallValue   = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: SmallStorageValueRlp,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+	}}
+	LargeStorageValue            = common.Hex2Bytes("00191b53778c567b14b50ba0000")
+	LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue)
+	storageWithLargeValue        = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: LargeStorageValueRlp,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+	}}
+	EmptyStorage        = make([]statediff.StorageDiff, 0)
+	StorageWithBadValue = statediff.StorageDiff{
+		Key:   StorageKey,
+		Value: []byte{0, 1, 2},
+		// this storage value will fail to be decoded as an RLP with the following error message:
+		// "input contains more than one value"
+	}
+	contractAddress        = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	ContractLeafKey        = crypto.Keccak256Hash(contractAddress[:])
+	anotherContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherContractLeafKey = crypto.Keccak256Hash(anotherContractAddress[:])
+
+	testAccount = state.Account{
+		Nonce:    NewNonceValue,
+		Balance:  big.NewInt(NewBalanceValue),
+		Root:     ContractRoot,
+		CodeHash: CodeHash,
+	}
+	valueBytes, _       = rlp.EncodeToBytes(testAccount)
+	CreatedAccountDiffs = []statediff.AccountDiff{
+		{
+			Key:     ContractLeafKey.Bytes(),
+			Value:   valueBytes,
+			Storage: storageWithSmallValue,
+		},
+	}
+
+	UpdatedAccountDiffs = []statediff.AccountDiff{{
+		Key:     AnotherContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storageWithLargeValue,
+	}}
+
+	DeletedAccountDiffs = []statediff.AccountDiff{{
+		Key:     AnotherContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storageWithSmallValue,
+	}}
+
+	MockStateDiff = statediff.StateDiff{
+		BlockNumber:     BlockNumber,
+		BlockHash:       common.HexToHash(BlockHash),
+		CreatedAccounts: CreatedAccountDiffs,
+		DeletedAccounts: DeletedAccountDiffs,
+		UpdatedAccounts: UpdatedAccountDiffs,
+	}
+	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
+
+	mockTransaction1 = types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil)
+	mockTransaction2 = types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil)
+	MockTransactions = types.Transactions{mockTransaction1, mockTransaction2}
+
+	mockReceipt1 = types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt2 = types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	MockReceipts = types.Receipts{mockReceipt1, mockReceipt2}
+
+	MockHeader = types.Header{
+		Time:        0,
+		Number:      BlockNumber,
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+	}
+	MockBlock       = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock)
+
+	MockStatediffPayload = statediff.Payload{
+		BlockRlp:     MockBlockRlp,
+		StateDiffRlp: MockStateDiffBytes,
+		Err:          nil,
+	}
+
+	EmptyStatediffPayload = statediff.Payload{
+		BlockRlp:     []byte{},
+		StateDiffRlp: []byte{},
+		Err:          nil,
+	}
+
+	ErrStatediffPayload = statediff.Payload{
+		BlockRlp:     []byte{},
+		StateDiffRlp: []byte{},
+		Err:          errors.New("mock error"),
+	}
+)
@@ -18,14 +18,13 @@ package transformer

 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )

 type StorageTransformer interface {
-	Execute(row utils.StorageDiffRow) error
-	ContractAddress() common.Address
+	Execute(diff utils.StorageDiff) error
+	KeccakContractAddress() common.Hash
 }

 type StorageTransformerInitializer func(db *postgres.DB) StorageTransformer
@@ -18,69 +18,76 @@ package watcher

 import (
 	"fmt"
-	"reflect"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"time"
 )

+type IStorageWatcher interface {
+	AddTransformers(initializers []transformer.StorageTransformerInitializer)
+	Execute(diffsChan chan utils.StorageDiff, errsChan chan error, queueRecheckInterval time.Duration)
+}
+
 type StorageWatcher struct {
-	db             *postgres.DB
-	StorageFetcher fetcher.IStorageFetcher
-	Queue          storage.IStorageQueue
-	Transformers   map[common.Address]transformer.StorageTransformer
+	db                        *postgres.DB
+	StorageFetcher            fetcher.IStorageFetcher
+	Queue                     storage.IStorageQueue
+	KeccakAddressTransformers map[common.Hash]transformer.StorageTransformer // keccak hash of an address => transformer
 }

 func NewStorageWatcher(fetcher fetcher.IStorageFetcher, db *postgres.DB) StorageWatcher {
-	transformers := make(map[common.Address]transformer.StorageTransformer)
 	queue := storage.NewStorageQueue(db)
+	transformers := make(map[common.Hash]transformer.StorageTransformer)
 	return StorageWatcher{
-		db:             db,
-		StorageFetcher: fetcher,
-		Queue:          queue,
-		Transformers:   transformers,
+		db:                        db,
+		StorageFetcher:            fetcher,
+		Queue:                     queue,
+		KeccakAddressTransformers: transformers,
 	}
 }

 func (storageWatcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
 		storageTransformer := initializer(storageWatcher.db)
-		storageWatcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
+		storageWatcher.KeccakAddressTransformers[storageTransformer.KeccakContractAddress()] = storageTransformer
 	}
 }

-func (storageWatcher StorageWatcher) Execute(rows chan utils.StorageDiffRow, errs chan error, queueRecheckInterval time.Duration) {
+func (storageWatcher StorageWatcher) Execute(diffsChan chan utils.StorageDiff, errsChan chan error, queueRecheckInterval time.Duration) {
 	ticker := time.NewTicker(queueRecheckInterval)
-	go storageWatcher.StorageFetcher.FetchStorageDiffs(rows, errs)
+	go storageWatcher.StorageFetcher.FetchStorageDiffs(diffsChan, errsChan)
 	for {
 		select {
-		case fetchErr := <-errs:
+		case fetchErr := <-errsChan:
 			logrus.Warn(fmt.Sprintf("error fetching storage diffs: %s", fetchErr))
-		case row := <-rows:
-			storageWatcher.processRow(row)
+		case diff := <-diffsChan:
+			storageWatcher.processRow(diff)
 		case <-ticker.C:
 			storageWatcher.processQueue()
 		}
 	}
 }

-func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
-	storageTransformer, ok := storageWatcher.Transformers[row.Contract]
+func (storageWatcher StorageWatcher) getTransformer(diff utils.StorageDiff) (transformer.StorageTransformer, bool) {
+	storageTransformer, ok := storageWatcher.KeccakAddressTransformers[diff.HashedAddress]
+	return storageTransformer, ok
+}
+
+func (storageWatcher StorageWatcher) processRow(diff utils.StorageDiff) {
+	storageTransformer, ok := storageWatcher.getTransformer(diff)
 	if !ok {
-		// ignore rows from unwatched contracts
+		logrus.Debug("ignoring a diff from an unwatched contract")
 		return
 	}
-	executeErr := storageTransformer.Execute(row)
+	executeErr := storageTransformer.Execute(diff)
 	if executeErr != nil {
 		logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
-		queueErr := storageWatcher.Queue.Add(row)
+		queueErr := storageWatcher.Queue.Add(diff)
 		if queueErr != nil {
 			logrus.Warn(fmt.Sprintf("error queueing storage diff: %s", queueErr))
 		}
@@ -88,20 +95,20 @@ func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
 }

 func (storageWatcher StorageWatcher) processQueue() {
-	rows, fetchErr := storageWatcher.Queue.GetAll()
+	diffs, fetchErr := storageWatcher.Queue.GetAll()
 	if fetchErr != nil {
 		logrus.Warn(fmt.Sprintf("error getting queued storage: %s", fetchErr))
 	}
-	for _, row := range rows {
-		storageTransformer, ok := storageWatcher.Transformers[row.Contract]
+	for _, diff := range diffs {
+		storageTransformer, ok := storageWatcher.getTransformer(diff)
 		if !ok {
-			// delete row from queue if address no longer watched
-			storageWatcher.deleteRow(row.Id)
+			// delete diff from queue if address no longer watched
+			storageWatcher.deleteRow(diff.Id)
 			continue
 		}
-		executeErr := storageTransformer.Execute(row)
+		executeErr := storageTransformer.Execute(diff)
 		if executeErr == nil {
-			storageWatcher.deleteRow(row.Id)
+			storageWatcher.deleteRow(diff.Id)
 		}
 	}
 }
@@ -109,10 +116,6 @@ func (storageWatcher StorageWatcher) processQueue() {
 func (storageWatcher StorageWatcher) deleteRow(id int) {
 	deleteErr := storageWatcher.Queue.Delete(id)
 	if deleteErr != nil {
-		logrus.Warn(fmt.Sprintf("error deleting persisted row from queue: %s", deleteErr))
+		logrus.Warn(fmt.Sprintf("error deleting persisted diff from queue: %s", deleteErr))
 	}
 }
-
-func isKeyNotFound(executeErr error) bool {
-	return reflect.TypeOf(executeErr) == reflect.TypeOf(utils.ErrStorageKeyNotFound{})
-}
@@ -17,59 +17,60 @@
 package watcher_test

 import (
-	"io/ioutil"
-	"os"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/test_config"
+	"io/ioutil"
+	"os"
+	"time"
 )

 var _ = Describe("Storage Watcher", func() {
-	It("adds transformers", func() {
-		fakeAddress := common.HexToAddress("0x12345")
-		fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
-		w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))
+	Describe("AddTransformer", func() {
+		It("adds transformers", func() {
+			fakeHashedAddress := utils.HexToKeccak256Hash("0x12345")
+			fakeTransformer := &mocks.MockStorageTransformer{KeccakOfAddress: fakeHashedAddress}
+			w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))

-		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
+			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})

-		Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
+			Expect(w.KeccakAddressTransformers[fakeHashedAddress]).To(Equal(fakeTransformer))
+		})
 	})

-	Describe("executing watcher", func() {
+	Describe("Execute", func() {
 		var (
 			errs            chan error
 			mockFetcher     *mocks.MockStorageFetcher
 			mockQueue       *mocks.MockStorageQueue
 			mockTransformer *mocks.MockStorageTransformer
-			row             utils.StorageDiffRow
-			rows            chan utils.StorageDiffRow
+			csvDiff         utils.StorageDiff
+			diffs           chan utils.StorageDiff
 			storageWatcher  watcher.StorageWatcher
+			hashedAddress   common.Hash
 		)

 		BeforeEach(func() {
 			errs = make(chan error)
-			rows = make(chan utils.StorageDiffRow)
-			address := common.HexToAddress("0x0123456789abcdef")
+			diffs = make(chan utils.StorageDiff)
+			hashedAddress = utils.HexToKeccak256Hash("0x0123456789abcdef")
 			mockFetcher = mocks.NewMockStorageFetcher()
 			mockQueue = &mocks.MockStorageQueue{}
-			mockTransformer = &mocks.MockStorageTransformer{Address: address}
-			row = utils.StorageDiffRow{
-				Id:           1337,
-				Contract:     address,
-				BlockHash:    common.HexToHash("0xfedcba9876543210"),
-				BlockHeight:  0,
-				StorageKey:   common.HexToHash("0xabcdef1234567890"),
-				StorageValue: common.HexToHash("0x9876543210abcdef"),
+			mockTransformer = &mocks.MockStorageTransformer{KeccakOfAddress: hashedAddress}
+			csvDiff = utils.StorageDiff{
+				Id:            1337,
+				HashedAddress: hashedAddress,
+				BlockHash:     common.HexToHash("0xfedcba9876543210"),
+				BlockHeight:   0,
+				StorageKey:    common.HexToHash("0xabcdef1234567890"),
+				StorageValue:  common.HexToHash("0x9876543210abcdef"),
 			}
 		})

@@ -83,7 +84,7 @@ var _ = Describe("Storage Watcher", func() {
 			defer os.Remove(tempFile.Name())
 			logrus.SetOutput(tempFile)

-			go storageWatcher.Execute(rows, errs, time.Hour)
+			go storageWatcher.Execute(diffs, errs, time.Hour)

 			Eventually(func() (string, error) {
 				logContent, err := ioutil.ReadFile(tempFile.Name())
@@ -92,39 +93,39 @@ var _ = Describe("Storage Watcher", func() {
 			close(done)
 		})

-		Describe("transforming new storage diffs", func() {
+		Describe("transforming new storage diffs from csv", func() {
 			BeforeEach(func() {
-				mockFetcher.RowsToReturn = []utils.StorageDiffRow{row}
+				mockFetcher.DiffsToReturn = []utils.StorageDiff{csvDiff}
 				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
 				storageWatcher.Queue = mockQueue
 				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
 			})

-			It("executes transformer for recognized storage row", func(done Done) {
-				go storageWatcher.Execute(rows, errs, time.Hour)
+			It("executes transformer for recognized storage diff", func(done Done) {
+				go storageWatcher.Execute(diffs, errs, time.Hour)

-				Eventually(func() utils.StorageDiffRow {
-					return mockTransformer.PassedRow
-				}).Should(Equal(row))
+				Eventually(func() utils.StorageDiff {
+					return mockTransformer.PassedDiff
+				}).Should(Equal(csvDiff))
 				close(done)
 			})

-			It("queues row for later processing if transformer execution fails", func(done Done) {
+			It("queues diff for later processing if transformer execution fails", func(done Done) {
 				mockTransformer.ExecuteErr = fakes.FakeError

-				go storageWatcher.Execute(rows, errs, time.Hour)
+				go storageWatcher.Execute(diffs, errs, time.Hour)

 				Expect(<-errs).To(BeNil())
 				Eventually(func() bool {
 					return mockQueue.AddCalled
 				}).Should(BeTrue())
-				Eventually(func() utils.StorageDiffRow {
-					return mockQueue.AddPassedRow
-				}).Should(Equal(row))
+				Eventually(func() utils.StorageDiff {
+					return mockQueue.AddPassedDiff
+				}).Should(Equal(csvDiff))
 				close(done)
 			})

-			It("logs error if queueing row fails", func(done Done) {
+			It("logs error if queueing diff fails", func(done Done) {
 				mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
 				mockQueue.AddError = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")
@@ -132,7 +133,7 @@ var _ = Describe("Storage Watcher", func() {
 				defer os.Remove(tempFile.Name())
 				logrus.SetOutput(tempFile)

-				go storageWatcher.Execute(rows, errs, time.Hour)
+				go storageWatcher.Execute(diffs, errs, time.Hour)

 				Eventually(func() bool {
 					return mockQueue.AddCalled
@@ -147,20 +148,38 @@ var _ = Describe("Storage Watcher", func() {

 		Describe("transforming queued storage diffs", func() {
 			BeforeEach(func() {
-				mockQueue.RowsToReturn = []utils.StorageDiffRow{row}
+				mockQueue.DiffsToReturn = []utils.StorageDiff{csvDiff}
 				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
 				storageWatcher.Queue = mockQueue
 				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
 			})

-			It("logs error if getting queued storage fails", func(done Done) {
-				mockQueue.GetAllErr = fakes.FakeError
+			It("executes transformer for storage diff", func(done Done) {
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
+
+				Eventually(func() utils.StorageDiff {
+					return mockTransformer.PassedDiff
+				}).Should(Equal(csvDiff))
+				close(done)
+			})
+
+			It("deletes diff from queue if transformer execution successful", func(done Done) {
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(csvDiff.Id))
+				close(done)
+			})
+
+			It("logs error if deleting persisted diff fails", func(done Done) {
+				mockQueue.DeleteErr = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")
 				Expect(fileErr).NotTo(HaveOccurred())
 				defer os.Remove(tempFile.Name())
 				logrus.SetOutput(tempFile)

-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)

 				Eventually(func() (string, error) {
 					logContent, err := ioutil.ReadFile(tempFile.Name())
@@ -169,68 +188,34 @@ var _ = Describe("Storage Watcher", func() {
 				close(done)
 			})

-			It("executes transformer for storage row", func(done Done) {
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+			It("deletes obsolete diff from queue if contract not recognized", func(done Done) {
+				obsoleteDiff := utils.StorageDiff{
+					Id:            csvDiff.Id + 1,
+					HashedAddress: utils.HexToKeccak256Hash("0xfedcba9876543210"),
+				}
+				mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}

-				Eventually(func() utils.StorageDiffRow {
-					return mockTransformer.PassedRow
-				}).Should(Equal(row))
-				close(done)
-			})
-
-			It("deletes row from queue if transformer execution successful", func(done Done) {
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)

 				Eventually(func() int {
 					return mockQueue.DeletePassedId
-				}).Should(Equal(row.Id))
+				}).Should(Equal(obsoleteDiff.Id))
 				close(done)
 			})

-			It("logs error if deleting persisted row fails", func(done Done) {
+			It("logs error if deleting obsolete diff fails", func(done Done) {
+				obsoleteDiff := utils.StorageDiff{
+					Id:            csvDiff.Id + 1,
+					HashedAddress: utils.HexToKeccak256Hash("0xfedcba9876543210"),
+				}
+				mockQueue.DiffsToReturn = []utils.StorageDiff{obsoleteDiff}
 				mockQueue.DeleteErr = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")
 				Expect(fileErr).NotTo(HaveOccurred())
 				defer os.Remove(tempFile.Name())
 				logrus.SetOutput(tempFile)

-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
-
-				Eventually(func() (string, error) {
-					logContent, err := ioutil.ReadFile(tempFile.Name())
-					return string(logContent), err
-				}).Should(ContainSubstring(fakes.FakeError.Error()))
-				close(done)
-			})
-
-			It("deletes obsolete row from queue if contract not recognized", func(done Done) {
-				obsoleteRow := utils.StorageDiffRow{
-					Id:       row.Id + 1,
-					Contract: common.HexToAddress("0xfedcba9876543210"),
-				}
-				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
-
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
-
-				Eventually(func() int {
-					return mockQueue.DeletePassedId
-				}).Should(Equal(obsoleteRow.Id))
-				close(done)
-			})
-
-			It("logs error if deleting obsolete row fails", func(done Done) {
-				obsoleteRow := utils.StorageDiffRow{
-					Id:       row.Id + 1,
-					Contract: common.HexToAddress("0xfedcba9876543210"),
-				}
-				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
-				mockQueue.DeleteErr = fakes.FakeError
-				tempFile, fileErr := ioutil.TempFile("", "log")
-				Expect(fileErr).NotTo(HaveOccurred())
-				defer os.Remove(tempFile.Name())
-				logrus.SetOutput(tempFile)
-
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+				go storageWatcher.Execute(diffs, errs, time.Nanosecond)

 				Eventually(func() (string, error) {
 					logContent, err := ioutil.ReadFile(tempFile.Name())
@@ -239,6 +224,5 @@ var _ = Describe("Storage Watcher", func() {
 				close(done)
 			})
 		})
-
 	})
 })
@@ -18,8 +18,8 @@ package retriever

 import (
 	"database/sql"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )

 // Block retriever is used to retrieve the first block for a given contract and the most recent block
@@ -55,7 +55,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
 // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
 func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
 	var firstBlock int64
-	addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr)
+	addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
 	if getAddressErr != nil {
 		return firstBlock, getAddressErr
 	}
@@ -72,10 +72,6 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
 	return firstBlock, err
 }

-func addressRepository() repositories.AddressRepository {
-	return repositories.AddressRepository{}
-}
-
 // In which case this servers as a heuristic to find the first block by finding the first contract event log
 func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64, error) {
 	var firstBlock int
@@ -19,6 +19,8 @@ package transformer
 import (
 	"errors"

+	"github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
@@ -106,7 +108,11 @@ func (tr *Transformer) Init() error {

 		// Get contract name if it has one
 		var name = new(string)
-		tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock)
+		pollingErr := tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, tr.LastBlock)
+		if pollingErr != nil {
+			// can't return this error because "name" might not exist on the contract
+			logrus.Warnf("error fetching contract data: %s", pollingErr.Error())
+		}

 		// Remove any potential accidental duplicate inputs in arg filter values
 		eventArgs := map[string]bool{}
@@ -132,7 +132,7 @@ func (c *Converter) Convert(logs []gethTypes.Log, event types.Event, headerID in

 // Convert the given watched event logs into types.Logs; returns a map of event names to a slice of their converted logs
 func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.Event, headerID int64) (map[string][]types.Log, error) {
-	contract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
+	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	eventsToLogs := make(map[string][]types.Log)
 	for _, event := range events {
 		eventsToLogs[event.Name] = make([]types.Log, 0, len(logs))
@@ -141,7 +141,7 @@ func (c *Converter) ConvertBatch(logs []gethTypes.Log, events map[string]types.E
 			// If the log is of this event type, process it as such
 			if event.Sig() == log.Topics[0] {
 				values := make(map[string]interface{})
-				err := contract.UnpackLogIntoMap(values, event.Name, log)
+				err := boundContract.UnpackLogIntoMap(values, event.Name, log)
 				if err != nil {
 					return nil, err
 				}
@@ -21,6 +21,7 @@ import (

 	"github.com/hashicorp/golang-lru"
 	"github.com/jmoiron/sqlx"
+	"github.com/sirupsen/logrus"

 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -148,7 +149,10 @@ func (r *headerRepository) MarkHeadersCheckedForAll(headers []core.Header, ids [
 		pgStr = pgStr[:len(pgStr)-2]
 		_, err = tx.Exec(pgStr, header.Id)
 		if err != nil {
-			tx.Rollback()
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transaction: %s", rollbackErr.Error())
+			}
 			return err
 		}
 	}
@@ -179,7 +183,7 @@ func (r *headerRepository) MissingHeaders(startingBlockNumber, endingBlockNumber
 				ORDER BY headers.block_number`
 		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
 	}
-	return contiguousHeaders(result, startingBlockNumber), err
+	return continuousHeaders(result), err
 }

 // Returns missing headers for all of the provided checked_headers column ids
@@ -207,26 +211,7 @@ func (r *headerRepository) MissingHeadersForAll(startingBlockNumber, endingBlock
 		query = baseQuery + endStr
 		err = r.db.Select(&result, query, startingBlockNumber, endingBlockNumber, r.db.Node.ID)
 	}
-	return contiguousHeaders(result, startingBlockNumber), err
-}
-
-// Returns a continuous set of headers that is contiguous with the provided startingBlockNumber
-func contiguousHeaders(headers []core.Header, startingBlockNumber int64) []core.Header {
-	if len(headers) < 1 {
-		return headers
-	}
-	previousHeader := headers[0].BlockNumber
-	if previousHeader != startingBlockNumber {
-		return []core.Header{}
-	}
-	for i := 1; i < len(headers); i++ {
-		previousHeader++
-		if headers[i].BlockNumber != previousHeader {
-			return headers[:i]
-		}
-	}
-
-	return headers
+	return continuousHeaders(result), err
 }

 // Returns headers that have been checked for all of the provided event ids but not for the provided method ids
@@ -265,6 +250,7 @@ func (r *headerRepository) MissingMethodsCheckedEventsIntersection(startingBlock
 // Returns a continuous set of headers
 func continuousHeaders(headers []core.Header) []core.Header {
 	if len(headers) < 1 {
+		logrus.Trace("no headers to arrange continuously")
 		return headers
 	}
 	previousHeader := headers[0].BlockNumber
@@ -120,7 +120,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 		})
@@ -130,16 +130,16 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

 			h1 := missingHeaders[0]
 			h2 := missingHeaders[1]
 			h3 := missingHeaders[2]
-			Expect(h1.BlockNumber).To(Equal(int64(6194632)))
-			Expect(h2.BlockNumber).To(Equal(int64(6194633)))
-			Expect(h3.BlockNumber).To(Equal(int64(6194634)))
+			Expect(h1.BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
+			Expect(h2.BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
+			Expect(h3.BlockNumber).To(Equal(mocks.MockHeader3.BlockNumber))
 		})

 		It("Returns only contiguous chunks of headers", func() {
@@ -147,11 +147,11 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632)))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633)))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
 		})

 		It("Fails if eventID does not yet exist in check_headers table", func() {
@@ -159,7 +159,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			_, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, "notEventId")
+			_, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, "notEventId")
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -170,14 +170,14 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

 			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

@@ -186,7 +186,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderChecked(missingHeaders[0].Id, eventIDs[2])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(6194633, 6194635, eventIDs)
+			missingHeaders, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
 		})
@@ -196,11 +196,23 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
-			Expect(missingHeaders[0].BlockNumber).To(Equal(int64(6194632)))
-			Expect(missingHeaders[1].BlockNumber).To(Equal(int64(6194633)))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader1.BlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader2.BlockNumber))
+		})
+
+		It("returns headers after starting header if starting header not missing", func() {
+			addLaterHeaders(coreHeaderRepo)
+			err := contractHeaderRepo.AddCheckColumns(eventIDs)
+			Expect(err).NotTo(HaveOccurred())
+
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, -1, eventIDs)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(missingHeaders)).To(Equal(2))
+			Expect(missingHeaders[0].BlockNumber).To(Equal(mocks.MockHeader3.BlockNumber))
+			Expect(missingHeaders[1].BlockNumber).To(Equal(mocks.MockHeader4.BlockNumber))
 		})

 		It("Fails if one of the eventIDs does not yet exist in check_headers table", func() {
@@ -209,7 +221,7 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			badEventIDs := append(eventIDs, "notEventId")

-			_, err = contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, badEventIDs)
+			_, err = contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, badEventIDs)
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -220,7 +232,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

@@ -228,7 +240,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderChecked(headerID, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0])
+			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
 		})
@@ -238,7 +250,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumn(eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

@@ -246,7 +258,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderChecked(headerID, "notEventId")
 			Expect(err).To(HaveOccurred())

-			missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))
 		})
@@ -258,7 +270,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.AddCheckColumns(eventIDs)
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(6194632, 6194635, eventIDs)
+			missingHeaders, err := contractHeaderRepo.MissingHeadersForAll(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

@@ -266,7 +278,7 @@ var _ = Describe("Repository", func() {
 			err = contractHeaderRepo.MarkHeaderCheckedForAll(headerID, eventIDs)
 			Expect(err).ToNot(HaveOccurred())

-			missingHeaders, err = contractHeaderRepo.MissingHeaders(6194633, 6194635, eventIDs[0])
+			missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader2.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(2))
 		})
@@ -285,7 +297,7 @@ var _ = Describe("Repository", func() {
 			for _, id := range methodIDs {
 				err := contractHeaderRepo.AddCheckColumn(id)
 				Expect(err).ToNot(HaveOccurred())
-				missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id)
+				missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, id)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(len(missingHeaders)).To(Equal(3))
 			}
@@ -293,7 +305,7 @@ var _ = Describe("Repository", func() {
 			err := contractHeaderRepo.MarkHeadersCheckedForAll(missingHeaders, methodIDs)
 			Expect(err).ToNot(HaveOccurred())
 			for _, id := range methodIDs {
-				missingHeaders, err = contractHeaderRepo.MissingHeaders(6194632, 6194635, id)
+				missingHeaders, err = contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, id)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(len(missingHeaders)).To(Equal(0))
 			}
@@ -310,7 +322,7 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 			}

-			missingHeaders, err := contractHeaderRepo.MissingHeaders(6194632, 6194635, eventIDs[0])
+			missingHeaders, err := contractHeaderRepo.MissingHeaders(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, eventIDs[0])
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(missingHeaders)).To(Equal(3))

@@ -325,23 +337,35 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 			}

-			intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(6194632, 6194635, methodIDs, eventIDs)
+			intersectionHeaders, err := contractHeaderRepo.MissingMethodsCheckedEventsIntersection(mocks.MockHeader1.BlockNumber, mocks.MockHeader4.BlockNumber, methodIDs, eventIDs)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(intersectionHeaders)).To(Equal(1))
 			Expect(intersectionHeaders[0].Id).To(Equal(headerID2))
-
 		})
 	})
 })

 func addHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
+	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
+	Expect(err).NotTo(HaveOccurred())
 }

 func addDiscontinuousHeaders(coreHeaderRepo repositories.HeaderRepository) {
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
-	coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
+	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader1)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader2)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func addLaterHeaders(coreHeaderRepo repositories.HeaderRepository) {
+	_, err := coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader3)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = coreHeaderRepo.CreateOrUpdateHeader(mocks.MockHeader4)
+	Expect(err).NotTo(HaveOccurred())
 }
@@ -44,9 +44,12 @@ var _ = Describe("Block Retriever", func() {

 	Describe("RetrieveFirstBlock", func() {
 		It("Retrieves block number of earliest header in the database", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			_, err := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			Expect(err).ToNot(HaveOccurred())

 			i, err := r.RetrieveFirstBlock()
 			Expect(err).NotTo(HaveOccurred())
@@ -61,9 +64,12 @@ var _ = Describe("Block Retriever", func() {

 	Describe("RetrieveMostRecentBlock", func() {
 		It("Retrieves the latest header's block number", func() {
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
-			headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			_, err := headerRepository.CreateOrUpdateHeader(mocks.MockHeader1)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader2)
+			Expect(err).ToNot(HaveOccurred())
+			_, err = headerRepository.CreateOrUpdateHeader(mocks.MockHeader3)
+			Expect(err).ToNot(HaveOccurred())

 			i, err := r.RetrieveMostRecentBlock()
 			Expect(err).ToNot(HaveOccurred())
@@ -18,10 +18,12 @@ package transformer

 import (
 	"errors"
+	"fmt"
 	"strings"

 	"github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/sirupsen/logrus"

 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
@@ -107,22 +109,22 @@ func (tr *Transformer) Init() error {
 		// Configure Abi
 		if tr.Config.Abis[contractAddr] == "" {
 			// If no abi is given in the config, this method will try fetching from internal look-up table and etherscan
-			err := tr.Parser.Parse(contractAddr)
-			if err != nil {
-				return err
+			parseErr := tr.Parser.Parse(contractAddr)
+			if parseErr != nil {
+				return fmt.Errorf("error parsing contract by address: %s", parseErr.Error())
 			}
 		} else {
 			// If we have an abi from the config, load that into the parser
-			err := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
-			if err != nil {
-				return err
+			parseErr := tr.Parser.ParseAbiStr(tr.Config.Abis[contractAddr])
+			if parseErr != nil {
+				return fmt.Errorf("error parsing contract abi: %s", parseErr.Error())
 			}
 		}

 		// Get first block and most recent block number in the header repo
-		firstBlock, err := tr.Retriever.RetrieveFirstBlock()
-		if err != nil {
-			return err
+		firstBlock, retrieveErr := tr.Retriever.RetrieveFirstBlock()
+		if retrieveErr != nil {
+			return fmt.Errorf("error retrieving first block: %s", retrieveErr.Error())
 		}

 		// Set to specified range if it falls within the bounds
@@ -132,7 +134,11 @@ func (tr *Transformer) Init() error {

 		// Get contract name if it has one
 		var name = new(string)
-		tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1)
+		pollingErr := tr.Poller.FetchContractData(tr.Parser.Abi(), contractAddr, "name", nil, name, -1)
+		if pollingErr != nil {
+			// can't return this error because "name" might not exist on the contract
+			logrus.Warnf("error fetching contract data: %s", pollingErr.Error())
+		}

 		// Remove any potential accidental duplicate inputs
 		eventArgs := map[string]bool{}
@@ -165,9 +171,9 @@ func (tr *Transformer) Init() error {
 		tr.sortedEventIds[con.Address] = make([]string, 0, len(con.Events))
 		for _, event := range con.Events {
 			eventId := strings.ToLower(event.Name + "_" + con.Address)
-			err := tr.HeaderRepository.AddCheckColumn(eventId)
-			if err != nil {
-				return err
+			addColumnErr := tr.HeaderRepository.AddCheckColumn(eventId)
+			if addColumnErr != nil {
+				return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
 			}
 			// Keep track of this event id; sorted and unsorted
 			tr.sortedEventIds[con.Address] = append(tr.sortedEventIds[con.Address], eventId)
@@ -180,9 +186,9 @@ func (tr *Transformer) Init() error {
 		tr.sortedMethodIds[con.Address] = make([]string, 0, len(con.Methods))
 		for _, m := range con.Methods {
 			methodId := strings.ToLower(m.Name + "_" + con.Address)
-			err := tr.HeaderRepository.AddCheckColumn(methodId)
-			if err != nil {
-				return err
+			addColumnErr := tr.HeaderRepository.AddCheckColumn(methodId)
+			if addColumnErr != nil {
+				return fmt.Errorf("error adding check column: %s", addColumnErr.Error())
 			}
 			tr.sortedMethodIds[con.Address] = append(tr.sortedMethodIds[con.Address], methodId)
 		}
@@ -202,9 +208,9 @@ func (tr *Transformer) Execute() error {
 	}

 	// Find unchecked headers for all events across all contracts; these are returned in asc order
-	missingHeaders, err := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds)
-	if err != nil {
-		return err
+	missingHeaders, missingHeadersErr := tr.HeaderRepository.MissingHeadersForAll(tr.Start, -1, tr.eventIds)
+	if missingHeadersErr != nil {
+		return fmt.Errorf("error getting missing headers: %s", missingHeadersErr.Error())
 	}

 	// Iterate over headers
@@ -216,23 +222,24 @@ func (tr *Transformer) Execute() error {
 		// Map to sort batch fetched logs by which contract they belong to, for post fetch processing
 		sortedLogs := make(map[string][]gethTypes.Log)
 		// And fetch all event logs across contracts at this header
-		allLogs, err := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header)
-		if err != nil {
-			return err
+		allLogs, fetchErr := tr.Fetcher.FetchLogs(tr.contractAddresses, tr.eventFilters, header)
+		if fetchErr != nil {
+			return fmt.Errorf("error fetching logs: %s", fetchErr.Error())
 		}

 		// If no logs are found mark the header checked for all of these eventIDs
 		// and continue to method polling and onto the next iteration
 		if len(allLogs) < 1 {
-			err = tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
-			if err != nil {
-				return err
+			markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, tr.eventIds)
+			if markCheckedErr != nil {
+				return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
 			}
-			err = tr.methodPolling(header, tr.sortedMethodIds)
-			if err != nil {
-				return err
+			pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
+			if pollingErr != nil {
+				return fmt.Errorf("error polling methods: %s", pollingErr.Error())
 			}
 			tr.Start = header.BlockNumber + 1 // Empty header; setup to start at the next header
+			logrus.Tracef("no logs found for block %d, continuing", header.BlockNumber)
 			continue
 		}

@@ -245,6 +252,7 @@ func (tr *Transformer) Execute() error {
 		// Process logs for each contract
 		for conAddr, logs := range sortedLogs {
 			if logs == nil {
+				logrus.Tracef("no logs found for contract %s at block %d, continuing", conAddr, header.BlockNumber)
 				continue
 			}
 			// Configure converter with this contract
@@ -252,34 +260,35 @@ func (tr *Transformer) Execute() error {
 			tr.Converter.Update(con)

 			// Convert logs into batches of log mappings (eventName => []types.Logs
-			convertedLogs, err := tr.Converter.ConvertBatch(logs, con.Events, header.Id)
-			if err != nil {
-				return err
+			convertedLogs, convertErr := tr.Converter.ConvertBatch(logs, con.Events, header.Id)
+			if convertErr != nil {
+				return fmt.Errorf("error converting logs: %s", convertErr.Error())
 			}
 			// Cycle through each type of event log and persist them
 			for eventName, logs := range convertedLogs {
 				// If logs for this event are empty, mark them checked at this header and continue
 				if len(logs) < 1 {
 					eventId := strings.ToLower(eventName + "_" + con.Address)
-					err = tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId)
-					if err != nil {
-						return err
+					markCheckedErr := tr.HeaderRepository.MarkHeaderChecked(header.Id, eventId)
+					if markCheckedErr != nil {
+						return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
 					}
+					logrus.Tracef("no logs found for event %s on contract %s at block %d, continuing", eventName, conAddr, header.BlockNumber)
 					continue
 				}
 				// If logs aren't empty, persist them
 				// Header is marked checked in the transactions
-				err = tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
-				if err != nil {
-					return err
+				persistErr := tr.EventRepository.PersistLogs(logs, con.Events[eventName], con.Address, con.Name)
+				if persistErr != nil {
+					return fmt.Errorf("error persisting logs: %s", persistErr.Error())
 				}
 			}
 		}

 		// Poll contracts at this block height
-		err = tr.methodPolling(header, tr.sortedMethodIds)
-		if err != nil {
-			return err
+		pollingErr := tr.methodPolling(header, tr.sortedMethodIds)
+		if pollingErr != nil {
+			return fmt.Errorf("error polling methods: %s", pollingErr.Error())
 		}
 		// Success; setup to start at the next header
 		tr.Start = header.BlockNumber + 1
@@ -294,19 +303,20 @@ func (tr *Transformer) methodPolling(header core.Header, sortedMethodIds map[str
 		// Skip method polling processes if no methods are specified
 		// Also don't try to poll methods below this contract's specified starting block
 		if len(con.Methods) == 0 || header.BlockNumber < con.StartingBlock {
+			logrus.Tracef("not polling contract: %s", con.Address)
 			continue
 		}

 		// Poll all methods for this contract at this header
-		err := tr.Poller.PollContractAt(*con, header.BlockNumber)
-		if err != nil {
-			return err
+		pollingErr := tr.Poller.PollContractAt(*con, header.BlockNumber)
+		if pollingErr != nil {
+			return fmt.Errorf("error polling contract %s: %s", con.Address, pollingErr.Error())
 		}

 		// Mark this header checked for the methods
-		err = tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, sortedMethodIds[con.Address])
-		if err != nil {
-			return err
+		markCheckedErr := tr.HeaderRepository.MarkHeaderCheckedForAll(header.Id, sortedMethodIds[con.Address])
+		if markCheckedErr != nil {
+			return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
 		}
 	}

@@ -68,7 +68,7 @@ var _ = Describe("Transformer", func() {
 			err := t.Init()

 			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
+			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		})
 	})

@@ -109,7 +109,7 @@ var _ = Describe("Transformer", func() {
 			err := t.Init()

 			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(fakes.FakeError))
+			Expect(err.Error()).To(ContainSubstring(fakes.FakeError.Error()))
 		})
 	})
 })
@@ -24,7 +24,7 @@ import (
 var SupportsInterfaceABI = `[{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]`

 // Individual event interfaces for constructing ABI from
-var SupportsInterace = `{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}`
+var SupportsInterface = `{"constant":true,"inputs":[{"name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}`
 var AddrChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"a","type":"address"}],"name":"AddrChanged","type":"event"}`
 var ContentChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"hash","type":"bytes32"}],"name":"ContentChanged","type":"event"}`
 var NameChangeInterface = `{"anonymous":false,"inputs":[{"indexed":true,"name":"node","type":"bytes32"},{"indexed":false,"name":"name","type":"string"}],"name":"NameChanged","type":"event"}`
@@ -47,7 +47,7 @@ func newFetcherError(err error, fetchMethod string) *fetcherError {

 // Fetcher struct
 type Fetcher struct {
-	BlockChain core.BlockChain // Underyling Blockchain
+	BlockChain core.BlockChain // Underlying Blockchain
 }

 // Fetcher error
@@ -115,9 +115,9 @@ func SetupDBandBC() (*postgres.DB, core.BlockChain) {
 	rpcClient := client.NewRpcClient(rawRpcClient, infuraIPC)
 	ethClient := ethclient.NewClient(rawRpcClient)
 	blockChainClient := client.NewEthClient(ethClient)
-	node := node.MakeNode(rpcClient)
+	madeNode := node.MakeNode(rpcClient)
 	transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+	blockChain := geth.NewBlockChain(blockChainClient, rpcClient, madeNode, transactionConverter)

 	db, err := postgres.NewDB(config.Database{
 		Hostname: "localhost",
@@ -324,7 +324,7 @@ var MockConfig = config.ContractConfig{
 		"0x1234567890abcdef": "fake_abi",
 	},
 	Events: map[string][]string{
-		"0x1234567890abcdef": []string{"Transfer"},
+		"0x1234567890abcdef": {"Transfer"},
 	},
 	Methods: map[string][]string{
 		"0x1234567890abcdef": nil,
@@ -35,7 +35,7 @@ var TusdConfig = config.ContractConfig{
 		tusd: "",
 	},
 	Events: map[string][]string{
-		tusd: []string{"Transfer"},
+		tusd: {"Transfer"},
 	},
 	Methods: map[string][]string{
 		tusd: nil,
@@ -60,7 +60,7 @@ var ENSConfig = config.ContractConfig{
 		ens: "",
 	},
 	Events: map[string][]string{
-		ens: []string{"NewOwner"},
+		ens: {"NewOwner"},
 	},
 	Methods: map[string][]string{
 		ens: nil,
@@ -87,8 +87,8 @@ var ENSandTusdConfig = config.ContractConfig{
 		tusd: "",
 	},
 	Events: map[string][]string{
-		ens:  []string{"NewOwner"},
-		tusd: []string{"Transfer"},
+		ens:  {"NewOwner"},
+		tusd: {"Transfer"},
 	},
 	Methods: map[string][]string{
 		ens:  nil,
@@ -22,6 +22,7 @@ import (
 	"strings"

 	"github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"

 	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
@@ -31,7 +32,7 @@ import (
 const (
 	// Number of contract address and method ids to keep in cache
 	contractCacheSize = 100
-	eventChacheSize   = 1000
+	eventCacheSize    = 1000
 )

 // Event repository is used to persist event data into custom tables
@@ -52,7 +53,7 @@ type eventRepository struct {

 func NewEventRepository(db *postgres.DB, mode types.Mode) *eventRepository {
 	ccs, _ := lru.New(contractCacheSize)
-	ecs, _ := lru.New(eventChacheSize)
+	ecs, _ := lru.New(eventCacheSize)
 	return &eventRepository{
 		db:      db,
 		mode:    mode,
@@ -68,14 +69,14 @@ func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, c
 	if len(logs) == 0 {
 		return errors.New("event repository error: passed empty logs slice")
 	}
-	_, err := r.CreateContractSchema(contractAddr)
-	if err != nil {
-		return err
+	_, schemaErr := r.CreateContractSchema(contractAddr)
+	if schemaErr != nil {
+		return fmt.Errorf("error creating schema for contract %s: %s", contractAddr, schemaErr.Error())
 	}

-	_, err = r.CreateEventTable(contractAddr, eventInfo)
-	if err != nil {
-		return err
+	_, tableErr := r.CreateEventTable(contractAddr, eventInfo)
+	if tableErr != nil {
+		return fmt.Errorf("error creating table for event %s on contract %s: %s", eventInfo.Name, contractAddr, tableErr.Error())
 	}

 	return r.persistLogs(logs, eventInfo, contractAddr, contractName)
@@ -97,9 +98,9 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c

 // Creates a custom postgres command to persist logs for the given event (compatible with header synced vDB)
 func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Beginx()
-	if err != nil {
-		return err
+	tx, txErr := r.db.Beginx()
+	if txErr != nil {
+		return fmt.Errorf("error beginning db transaction: %s", txErr.Error())
 	}

 	for _, event := range logs {
@@ -130,20 +131,27 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type
 		}
 		pgStr = pgStr + ") ON CONFLICT DO NOTHING"

+		logrus.Tracef("query for inserting log: %s", pgStr)
 		// Add this query to the transaction
-		_, err = tx.Exec(pgStr, data...)
-		if err != nil {
-			tx.Rollback()
-			return err
+		_, execErr := tx.Exec(pgStr, data...)
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transactions while persisting logs: %s", rollbackErr.Error())
+			}
+			return fmt.Errorf("error executing query: %s", execErr.Error())
 		}
 	}

 	// Mark header as checked for this eventId
 	eventId := strings.ToLower(eventInfo.Name + "_" + contractAddr)
-	err = repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
-	if err != nil {
-		tx.Rollback()
-		return err
+	markCheckedErr := repository.MarkContractWatcherHeaderCheckedInTransaction(logs[0].Id, tx, eventId) // This assumes all logs are from same block
+	if markCheckedErr != nil {
+		rollbackErr := tx.Rollback()
+		if rollbackErr != nil {
+			logrus.Warnf("error rolling back transaction while marking header checked: %s", rollbackErr.Error())
+		}
+		return fmt.Errorf("error marking header checked: %s", markCheckedErr.Error())
 	}

 	return tx.Commit()
@@ -151,9 +159,9 @@ func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo type

 // Creates a custom postgres command to persist logs for the given event (compatible with fully synced vDB)
 func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
-	tx, err := r.db.Beginx()
-	if err != nil {
-		return err
+	tx, txErr := r.db.Beginx()
+	if txErr != nil {
+		return fmt.Errorf("error beginning db transaction: %s", txErr.Error())
 	}

 	for _, event := range logs {
@@ -179,10 +187,14 @@ func (r *eventRepository) persistFullSyncLogs(logs []types.Log, eventInfo types.
 		}
 		pgStr = pgStr + ") ON CONFLICT (vulcanize_log_id) DO NOTHING"

-		_, err = tx.Exec(pgStr, data...)
-		if err != nil {
-			tx.Rollback()
-			return err
+		logrus.Tracef("query for inserting log: %s", pgStr)
+		_, execErr := tx.Exec(pgStr, data...)
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transactions while persisting logs: %s", rollbackErr.Error())
+			}
+			return fmt.Errorf("error executing query: %s", execErr.Error())
 		}
 	}

@@ -198,15 +210,15 @@ func (r *eventRepository) CreateEventTable(contractAddr string, event types.Even
 	if ok {
 		return false, nil
 	}
-	tableExists, err := r.checkForTable(contractAddr, event.Name)
-	if err != nil {
-		return false, err
+	tableExists, checkTableErr := r.checkForTable(contractAddr, event.Name)
+	if checkTableErr != nil {
+		return false, fmt.Errorf("error checking for table: %s", checkTableErr)
 	}

 	if !tableExists {
-		err = r.newEventTable(tableID, event)
-		if err != nil {
-			return false, err
+		createTableErr := r.newEventTable(tableID, event)
+		if createTableErr != nil {
+			return false, fmt.Errorf("error creating table: %s", createTableErr.Error())
 		}
 	}

@@ -270,14 +282,14 @@ func (r *eventRepository) CreateContractSchema(contractAddr string) (bool, error
 	if ok {
 		return false, nil
 	}
-	schemaExists, err := r.checkForSchema(contractAddr)
-	if err != nil {
-		return false, err
+	schemaExists, checkSchemaErr := r.checkForSchema(contractAddr)
+	if checkSchemaErr != nil {
+		return false, fmt.Errorf("error checking for schema: %s", checkSchemaErr.Error())
 	}
 	if !schemaExists {
-		err = r.newContractSchema(contractAddr)
-		if err != nil {
-			return false, err
+		createSchemaErr := r.newContractSchema(contractAddr)
+		if createSchemaErr != nil {
+			return false, fmt.Errorf("error creating schema: %s", createSchemaErr.Error())
 		}
 	}

@@ -190,7 +190,7 @@ var _ = Describe("Repository", func() {
 				}
 				Expect(scanLog).To(Equal(expectedLog))

-				// Attempt to persist the same log again in seperate call
+				// Attempt to persist the same log again in separate call
 				err = dataStore.PersistLogs([]types.Log{*log}, event, con.Address, con.Name)
 				Expect(err).ToNot(HaveOccurred())

@@ -22,6 +22,7 @@ import (
 	"strings"

 	"github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"

 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/types"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -112,7 +113,10 @@ func (r *methodRepository) persistResults(results []types.Result, methodInfo typ
 		// Add this query to the transaction
 		_, err = tx.Exec(pgStr, data...)
 		if err != nil {
-			tx.Rollback()
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Warnf("error rolling back transaction: %s", rollbackErr.Error())
+			}
 			return err
 		}
 	}
@@ -48,12 +48,10 @@ var mockEvent = core.WatchedEvent{
 var _ = Describe("Address Retriever Test", func() {
 	var db *postgres.DB
 	var dataStore repository.EventRepository
-	var err error
 	var info *contract.Contract
 	var vulcanizeLogId int64
 	var log *types.Log
 	var r retriever.AddressRetriever
-	var addresses map[common.Address]bool
 	var wantedEvents = []string{"Transfer"}

 	BeforeEach(func() {
@@ -61,17 +59,18 @@ var _ = Describe("Address Retriever Test", func() {
 		mockEvent.LogID = vulcanizeLogId

 		event := info.Events["Transfer"]
-		err = info.GenerateFilters()
-		Expect(err).ToNot(HaveOccurred())
+		filterErr := info.GenerateFilters()
+		Expect(filterErr).ToNot(HaveOccurred())

 		c := converter.Converter{}
 		c.Update(info)
-		log, err = c.Convert(mockEvent, event)
-		Expect(err).ToNot(HaveOccurred())
+		var convertErr error
+		log, convertErr = c.Convert(mockEvent, event)
+		Expect(convertErr).ToNot(HaveOccurred())

 		dataStore = repository.NewEventRepository(db, types.FullSync)
-		dataStore.PersistLogs([]types.Log{*log}, event, info.Address, info.Name)
-		Expect(err).ToNot(HaveOccurred())
+		persistErr := dataStore.PersistLogs([]types.Log{*log}, event, info.Address, info.Name)
+		Expect(persistErr).ToNot(HaveOccurred())

 		r = retriever.NewAddressRetriever(db, types.FullSync)
 	})
@@ -82,8 +81,8 @@ var _ = Describe("Address Retriever Test", func() {

 	Describe("RetrieveTokenHolderAddresses", func() {
 		It("Retrieves a list of token holder addresses from persisted event logs", func() {
-			addresses, err = r.RetrieveTokenHolderAddresses(*info)
-			Expect(err).ToNot(HaveOccurred())
+			addresses, retrieveErr := r.RetrieveTokenHolderAddresses(*info)
+			Expect(retrieveErr).ToNot(HaveOccurred())

 			_, ok := addresses[common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")]
 			Expect(ok).To(Equal(true))
@@ -100,8 +99,8 @@ var _ = Describe("Address Retriever Test", func() {
 		})

 		It("Returns empty list when empty contract info is used", func() {
-			addresses, err = r.RetrieveTokenHolderAddresses(contract.Contract{})
-			Expect(err).ToNot(HaveOccurred())
+			addresses, retrieveErr := r.RetrieveTokenHolderAddresses(contract.Contract{})
+			Expect(retrieveErr).ToNot(HaveOccurred())
 			Expect(len(addresses)).To(Equal(0))
 		})
 	})
@@ -92,5 +92,5 @@ func (e Event) Sig() common.Hash {
 		types[i] = input.Type.String()
 	}

-	return common.BytesToHash(crypto.Keccak256([]byte(fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ",")))))
+	return crypto.Keccak256Hash([]byte(fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ","))))
 }
@@ -107,5 +107,5 @@ func (m Method) Sig() common.Hash {
 		i++
 	}

-	return common.BytesToHash(crypto.Keccak256([]byte(fmt.Sprintf("%v(%v)", m.Name, strings.Join(types, ",")))))
+	return crypto.Keccak256Hash([]byte(fmt.Sprintf("%v(%v)", m.Name, strings.Join(types, ","))))
 }
@@ -18,6 +18,7 @@ package core

 import (
 	"context"
+	"github.com/ethereum/go-ethereum/rpc"

 	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
 )
@@ -27,4 +28,5 @@ type RpcClient interface {
 	BatchCall(batch []client.BatchElem) error
 	IpcPath() string
 	SupportedModules() (map[string]string, error)
+	Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error)
 }
@@ -1,62 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repositories
-
-import (
-	"database/sql"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/jmoiron/sqlx"
-
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type AddressRepository struct{}
-
-func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
-	stringAddressToCommonAddress := common.HexToAddress(address)
-	hexAddress := stringAddressToCommonAddress.Hex()
-
-	var addressId int64
-	getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
-	if getErr == sql.ErrNoRows {
-		insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
-		return addressId, insertErr
-	}
-
-	return addressId, getErr
-}
-
-func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
-	stringAddressToCommonAddress := common.HexToAddress(address)
-	hexAddress := stringAddressToCommonAddress.Hex()
-
-	var addressId int64
-	getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
-	if getErr == sql.ErrNoRows {
-		insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
-		return addressId, insertErr
-	}
-
-	return addressId, getErr
-}
-
-func (AddressRepository) GetAddressById(db *postgres.DB, id int64) (string, error) {
-	var address string
-	getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
-	return address, getErr
-}
@@ -20,6 +20,7 @@ import (
 	"database/sql"
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -84,7 +85,7 @@ func createLogs(logs []core.FullSyncLog, receiptId int64, tx *sqlx.Tx) error {

 func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
 	var receiptId int64
-	addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
+	addressId, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
 	if getAddressErr != nil {
 		logrus.Error("createReceipt: Error getting address id: ", getAddressErr)
 		return receiptId, getAddressErr
@@ -22,6 +22,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
@@ -31,14 +32,12 @@ const insertHeaderSyncLogQuery = `INSERT INTO header_sync_logs
 		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING`

 type HeaderSyncLogRepository struct {
-	db                *postgres.DB
-	addressRepository AddressRepository
+	db *postgres.DB
 }

 func NewHeaderSyncLogRepository(db *postgres.DB) HeaderSyncLogRepository {
 	return HeaderSyncLogRepository{
-		db:                db,
-		addressRepository: AddressRepository{},
+		db: db,
 	}
 }

@@ -57,8 +56,8 @@ type headerSyncLog struct {
 	Raw         []byte
 }

-func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
-	rows, queryErr := repository.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`)
+func (repo HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]core.HeaderSyncLog, error) {
+	rows, queryErr := repo.db.Queryx(`SELECT * FROM public.header_sync_logs WHERE transformed = false`)
 	if queryErr != nil {
 		return nil, queryErr
 	}
@@ -74,7 +73,7 @@ func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]co
 		for _, topic := range rawLog.Topics {
 			logTopics = append(logTopics, common.BytesToHash(topic))
 		}
-		address, addrErr := repository.addressRepository.GetAddressById(repository.db, rawLog.Address)
+		address, addrErr := repository.GetAddressById(repo.db, rawLog.Address)
 		if addrErr != nil {
 			return nil, addrErr
 		}
@@ -104,13 +103,13 @@ func (repository HeaderSyncLogRepository) GetUntransformedHeaderSyncLogs() ([]co
 	return results, nil
 }

-func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
-	tx, txErr := repository.db.Beginx()
+func (repo HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, logs []types.Log) error {
+	tx, txErr := repo.db.Beginx()
 	if txErr != nil {
 		return txErr
 	}
 	for _, log := range logs {
-		err := repository.insertLog(headerID, log, tx)
+		err := repo.insertLog(headerID, log, tx)
 		if err != nil {
 			rollbackErr := tx.Rollback()
 			if rollbackErr != nil {
@@ -122,13 +121,13 @@ func (repository HeaderSyncLogRepository) CreateHeaderSyncLogs(headerID int64, l
 	return tx.Commit()
 }

-func (repository HeaderSyncLogRepository) insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error {
+func (repo HeaderSyncLogRepository) insertLog(headerID int64, log types.Log, tx *sqlx.Tx) error {
 	topics := buildTopics(log)
 	raw, jsonErr := log.MarshalJSON()
 	if jsonErr != nil {
 		return jsonErr
 	}
-	addressID, addrErr := repository.addressRepository.GetOrCreateAddressInTransaction(tx, log.Address.Hex())
+	addressID, addrErr := repository.GetOrCreateAddressInTransaction(tx, log.Address.Hex())
 	if addrErr != nil {
 		return addrErr
 	}
@@ -22,6 +22,7 @@ import (
 	"github.com/lib/pq"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	repository2 "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -79,8 +80,7 @@ var _ = Describe("Header sync log repository", func() {
 			Expect(lookupErr).NotTo(HaveOccurred())
 			Expect(dbLog.ID).NotTo(BeZero())
 			Expect(dbLog.HeaderID).To(Equal(headerID))
-			addressRepository := repositories.AddressRepository{}
-			actualAddress, addressErr := addressRepository.GetAddressById(db, dbLog.Address)
+			actualAddress, addressErr := repository2.GetAddressById(db, dbLog.Address)
 			Expect(addressErr).NotTo(HaveOccurred())
 			Expect(actualAddress).To(Equal(log.Address.Hex()))
 			Expect(dbLog.Topics[0]).To(Equal(log.Topics[0].Bytes()))
@@ -128,8 +128,7 @@ var _ = Describe("Header sync log repository", func() {
 				logTopics = append(logTopics, common.BytesToHash(topic))
 			}

-			addressRepository := repositories.AddressRepository{}
-			actualAddress, addressErr := addressRepository.GetAddressById(db, dbLog.Address)
+			actualAddress, addressErr := repository2.GetAddressById(db, dbLog.Address)
 			Expect(addressErr).NotTo(HaveOccurred())
 			reconstructedLog := types.Log{
 				Address:     common.HexToAddress(actualAddress),
@@ -19,6 +19,7 @@ package repositories
 import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/jmoiron/sqlx"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )

@@ -26,7 +27,7 @@ type HeaderSyncReceiptRepository struct{}

 func (HeaderSyncReceiptRepository) CreateHeaderSyncReceiptInTx(headerID, transactionID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
 	var receiptId int64
-	addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
+	addressId, getAddressErr := repository.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
 	if getAddressErr != nil {
 		log.Error("createReceipt: Error getting address id: ", getAddressErr)
 		return receiptId, getAddressErr
@@ -18,10 +18,13 @@ package fakes

 import (
 	"context"
+	"errors"
+	"github.com/ethereum/go-ethereum/statediff"
 	"math/big"

 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
 	. "github.com/onsi/gomega"

 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -29,18 +32,44 @@ import (
 )

 type MockRpcClient struct {
-	callContextErr   error
-	ipcPath          string
-	nodeType         core.NodeType
-	passedContext    context.Context
-	passedMethod     string
-	passedResult     interface{}
-	passedBatch      []client.BatchElem
-	lengthOfBatch    int
-	returnPOAHeader  core.POAHeader
-	returnPOAHeaders []core.POAHeader
-	returnPOWHeaders []*types.Header
-	supportedModules map[string]string
+	callContextErr      error
+	ipcPath             string
+	nodeType            core.NodeType
+	passedContext       context.Context
+	passedMethod        string
+	passedResult        interface{}
+	passedBatch         []client.BatchElem
+	passedNamespace     string
+	passedPayloadChan   chan statediff.Payload
+	passedSubscribeArgs []interface{}
+	lengthOfBatch       int
+	returnPOAHeader     core.POAHeader
+	returnPOAHeaders    []core.POAHeader
+	returnPOWHeaders    []*types.Header
+	supportedModules    map[string]string
+}
+
+func (client *MockRpcClient) Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error) {
+	client.passedNamespace = namespace
+
+	passedPayloadChan, ok := payloadChan.(chan statediff.Payload)
+	if !ok {
+		return nil, errors.New("passed in channel is not of the correct type")
+	}
+	client.passedPayloadChan = passedPayloadChan
+
+	for _, arg := range args {
+		client.passedSubscribeArgs = append(client.passedSubscribeArgs, arg)
+	}
+
+	subscription := rpc.ClientSubscription{}
+	return &subscription, nil
+}
+
+func (client *MockRpcClient) AssertSubscribeCalledWith(namespace string, payloadChan chan statediff.Payload, args []interface{}) {
+	Expect(client.passedNamespace).To(Equal(namespace))
+	Expect(client.passedPayloadChan).To(Equal(payloadChan))
+	Expect(client.passedSubscribeArgs).To(Equal(args))
 }

 func NewMockRpcClient() *MockRpcClient {
@@ -18,6 +18,9 @@ package client

 import (
 	"context"
+	"errors"
+	"reflect"
+
 	"github.com/ethereum/go-ethereum/rpc"
 )

@@ -73,3 +76,16 @@ func (client RpcClient) BatchCall(batch []BatchElem) error {
 	}
 	return client.client.BatchCall(rpcBatch)
 }
+
+// Subscribe subscribes to an rpc "namespace_subscribe" subscription with the given channel
+// The first argument needs to be the method we wish to invoke
+func (client RpcClient) Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error) {
+	chanVal := reflect.ValueOf(payloadChan)
+	if chanVal.Kind() != reflect.Chan || chanVal.Type().ChanDir()&reflect.SendDir == 0 {
+		return nil, errors.New("second argument to Subscribe must be a writable channel")
+	}
+	if chanVal.IsNil() {
+		return nil, errors.New("channel given to Subscribe must not be nil")
+	}
+	return client.client.Subscribe(context.Background(), namespace, payloadChan, args...)
+}
Author	SHA1	Message	Date
Elizabeth	f6ab9382b2	Address repo updates (#134 ) * Factor out get or create address into one sql string * Factor out getChecksumAddress method in address repo * Update address repo methods to not need a receiver * Move address repository to libraries/shared	2019-10-03 11:17:08 -05:00
Rob Mulholand	031043130e	Merge pull request #145 from vulcanize/contract-watcher-logging Increase logging in contract watcher	2019-10-02 16:30:17 -05:00
Rob Mulholand	4505382590	Increase logging in contract watcher - Focus on header mode - Add context to errors, trace guard clauses, warn on non-returned errors - Give errors distinct names so compiler will recognize if unchecked - Remove redundant type declarations/fix typos	2019-10-02 15:14:42 -05:00
Elizabeth	b4e16c4af5	Merge pull request #116 from vulcanize/storage-diffs-over-rpc Storage diffs over rpc	2019-10-02 09:56:29 -05:00
Elizabeth Engelman	f0d2741dea	Allow storageDiff source to be set through an env variable	2019-10-01 10:24:53 -05:00
Elizabeth Engelman	267de00f99	Update dependencies specifically golang.org/x/crypto/sha3	2019-09-26 10:41:12 -05:00
Elizabeth Engelman	5c0e5592ab	Updates after rebasing	2019-09-25 16:52:42 -05:00
Elizabeth Engelman	9c6182c356	Rename StorageDiff field from KeccakOfContractAddress to HashedAddress	2019-09-25 16:36:44 -05:00
Elizabeth Engelman	6672ecf547	Add trace debugging after parsing statediff	2019-09-25 16:36:44 -05:00
Elizabeth Engelman	f315988507	Factor out a bad storage diff for testing	2019-09-25 16:36:44 -05:00
Elizabeth Engelman	6869330bd3	Decode storage value RLP after fetching from statediffs	2019-09-25 16:36:44 -05:00
Elizabeth Engelman	2931edc317	Pin urfave/cli version to 1.20	2019-09-25 16:36:44 -05:00
Elizabeth Engelman	4166fb24dd	Replace any geth version with the vdb statediff patch	2019-09-25 16:36:08 -05:00
Elizabeth Engelman	e1de3afdfc	Small changes to helper method names/arg names	2019-09-25 16:36:08 -05:00
Rob Mulholand	2ff88de859	Extract helper for converting hex to keccak256 hash - Also prefer crypto.Keccak256Hash(x) to common.BytesToHash(crypto.Keccak256(x))	2019-09-25 16:36:08 -05:00
Rob Mulholand	d06dddbfaa	Always hash storage diff contract addresses - Enables syncing Geth and Parity diffs with same transformer lookup - Maybe worth always hashing the storage key so we don't need a hashed and not-hashed version in the key lookups?	2019-09-25 16:36:08 -05:00
Rob Mulholand	f574407bb6	Revert "Update Mappings interface to take a diff as an argument to Lookup" This reverts commit cda646b78ceb6714b132891ea4489a8ff4fc95e9.	2019-09-25 16:34:31 -05:00
Rob Mulholand	8111f4ec5e	Add function for adding hashed keys to mapping - Tool to facilitate parsing diffs from Geth patch that emits hashed versions of storage keys	2019-09-25 16:34:31 -05:00
Elizabeth Engelman	3fb8e13979	Update Mappings interface to take a diff as an argument to Lookup	2019-09-25 16:34:31 -05:00
Elizabeth Engelman	deb0315a49	Remove Gopkg files	2019-09-25 16:34:31 -05:00
Elizabeth Engelman	66d695d93b	Update keccak of address to be a common.Hash instead of a common.Address	2019-09-25 16:34:31 -05:00
Elizabeth Engelman	a8a8fe4ac2	Remove vendor dir	2019-09-25 16:34:31 -05:00
Elizabeth Engelman	9831badc2a	Use geth patch in go mod	2019-09-25 16:33:58 -05:00
Elizabeth Engelman	8e166f2a56	Update after rebasing with staging	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	639c561fdf	Remove unncessary pointer receiver from NewGethRpcStorageFetcher.FetchStorageDiffs	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	9226e53e0b	Panic if connecting to geth subscription fails	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	0cc90a1f80	Factor out an IStorageWatcher interface	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	18f47b7b6a	Memoizing keccak hhash of address => transformer	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	045d78be25	Rename StorageDiffRow -> StorageDiff	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	87252f42b9	Add additional logging	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	e637024cce	Update vendor dir	2019-09-25 16:32:31 -05:00
Elizabeth Engelman	7cd66b8775	Use a for loop for getting diffs off the payload channel	2019-09-25 16:32:30 -05:00
Elizabeth Engelman	7cbad1d89f	Override protobuf to version 3	2019-09-25 16:32:30 -05:00
Elizabeth Engelman	963479fdc0	Remove ipfs and eth-block-extractor as dependencies	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	b09b8a8735	Use geth state diff source in execute command	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	70fcc22a00	Apply go formatting	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	a577811e0a	Use geth state diff source in composeAndExecute	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	8c4a4d6587	Handle different contract address format for geth vs csv	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	ee244ac6f5	Apply go fmt	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	dc06991605	Wire up the streamer with a fetcher	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	1b4a901892	Rename storage_fetcher -> csv_tail_storage_fetcher	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	9de631f3a7	Add a statediff streamer	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	34f8e16c11	Add a Subscribe method to rpc client interface	2019-09-25 16:32:27 -05:00
Elizabeth Engelman	36533f7c3f	Update vendor directory and make necessary code changes Fixes for new geth version	2019-09-25 16:32:27 -05:00
grizz	20ce0ab852	Revert "option to write the plugin .go file and run the plugin migrations without building the .so file so that it can be done from the shell instead to afford complete error messages" This reverts commit `9bb2f27a69`.	2019-09-24 23:54:25 +00:00
Ian Norden	9bb2f27a69	option to write the plugin .go file and run the plugin migrations without building the .so file so that it can be done from the shell instead to afford complete error messages	2019-09-24 18:44:04 -05:00
Rob Mulholand	ba4e79fc63	Merge pull request #143 from vulcanize/contract-watcher-missing-headers Correctly identify missing headers after contract watcher restart	2019-09-24 08:55:25 -05:00
Rob Mulholand	c21c069b69	Replace magic numbers with variables	2019-09-23 16:13:45 -05:00
Rob Mulholand	eea0b0bdca	Return missing headers after starting block number - Previously, we required that the first missing header match the configured starting block number. This helps guarantee that we have the necessary data for method polling in memory, but prevents the process from moving forward if restarted after the starting block has already been checked.	2019-09-23 16:13:45 -05:00