Merge pull request #150 from vulcanize/unique-headers-constraint-v2

Add constraint to prevent duplicate headers

.travis.yml

@@ -1,27 +1,24 @@
 dist: trusty
 language: go
 go:
-  - 1.11
+  - 1.12
 services:
   - postgresql
 addons:
-  postgresql: '9.6'
+  postgresql: '11.2'
 go_import_path: github.com/vulcanize/vulcanizedb
 before_install:
   - make installtools
-  - bash ./scripts/install-postgres-10.sh
+  - bash ./scripts/install-postgres-11.sh
   - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
   - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
   - sudo apt-get update && sudo apt-get install yarn
 before_script:
   - sudo -u postgres createdb vulcanize_private
   - make migrate NAME=vulcanize_private
-  - cd postgraphile && yarn
 script:
-  - yarn test
+  - env GO111MODULE=on make test
-  - cd ../
+  - env GO111MODULE=on make integrationtest
-  - make test
-  - make integrationtest
 notifications:
   email: false
 env:

Gopkg.lock

@@ -1,563 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:48a213e9dc4880bbbd6999309a476fa4d3cc67560aa7127154cf8ea95bd464c2"
-  name = "github.com/allegro/bigcache"
-  packages = [
-    ".",
-    "queue",
-  ]
-  pruneopts = ""
-  revision = "f31987a23e44c5121ef8c8b2f2ea2e8ffa37b068"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
-  name = "github.com/aristanetworks/goarista"
-  packages = ["monotime"]
-  pruneopts = ""
-  revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
-  name = "github.com/btcsuite/btcd"
-  packages = ["btcec"]
-  pruneopts = ""
-  revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
-
-[[projects]]
-  digest = "1:5d47691333460db6ac83ced03c79b4bdb9aff3e322be24affb7855bed8affc6c"
-  name = "github.com/dave/jennifer"
-  packages = ["jen"]
-  pruneopts = ""
-  revision = "14e399b6b5e8456c66c45c955fc27b568bacb5c9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
-  name = "github.com/deckarep/golang-set"
-  packages = ["."]
-  pruneopts = ""
-  revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
-  version = "v1.7.1"
-
-[[projects]]
-  digest = "1:90d36f5b581e95e00ced808cd48824ed6c320c25887828cce461bdef4cb7bc7c"
-  name = "github.com/ethereum/go-ethereum"
-  packages = [
-    ".",
-    "accounts",
-    "accounts/abi",
-    "accounts/abi/bind",
-    "accounts/keystore",
-    "common",
-    "common/bitutil",
-    "common/hexutil",
-    "common/math",
-    "common/mclock",
-    "common/prque",
-    "core/rawdb",
-    "core/types",
-    "crypto",
-    "crypto/ecies",
-    "crypto/secp256k1",
-    "ethclient",
-    "ethdb",
-    "ethdb/leveldb",
-    "ethdb/memorydb",
-    "event",
-    "log",
-    "metrics",
-    "p2p",
-    "p2p/discover",
-    "p2p/discv5",
-    "p2p/enode",
-    "p2p/enr",
-    "p2p/nat",
-    "p2p/netutil",
-    "params",
-    "rlp",
-    "rpc",
-    "trie",
-  ]
-  pruneopts = ""
-  revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
-
-[[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  version = "v1.4.7"
-
-[[projects]]
-  digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
-  name = "github.com/go-stack/stack"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = ""
-  revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
-
-[[projects]]
-  digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = ""
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
-  name = "github.com/hashicorp/hcl"
-  packages = [
-    ".",
-    "hcl/ast",
-    "hcl/parser",
-    "hcl/printer",
-    "hcl/scanner",
-    "hcl/strconv",
-    "hcl/token",
-    "json/parser",
-    "json/scanner",
-    "json/token",
-  ]
-  pruneopts = ""
-  revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
-  name = "github.com/hpcloud/tail"
-  packages = [
-    ".",
-    "ratelimiter",
-    "util",
-    "watch",
-    "winfile",
-  ]
-  pruneopts = ""
-  revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
-  name = "github.com/huin/goupnp"
-  packages = [
-    ".",
-    "dcps/internetgateway1",
-    "dcps/internetgateway2",
-    "httpu",
-    "scpd",
-    "soap",
-    "ssdp",
-  ]
-  pruneopts = ""
-  revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
-
-[[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = ""
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
-  name = "github.com/jackpal/go-nat-pmp"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
-  name = "github.com/jmoiron/sqlx"
-  packages = [
-    ".",
-    "reflectx",
-  ]
-  pruneopts = ""
-  revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
-
-[[projects]]
-  digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3"
-  name = "github.com/konsorten/go-windows-terminal-sequences"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
-  name = "github.com/lib/pq"
-  packages = [
-    ".",
-    "oid",
-  ]
-  pruneopts = ""
-  revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
-
-[[projects]]
-  digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2353362d570a7bfa228149c62842019201cfb71"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  pruneopts = ""
-  revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
-  name = "github.com/onsi/ginkgo"
-  packages = [
-    ".",
-    "config",
-    "internal/codelocation",
-    "internal/containernode",
-    "internal/failer",
-    "internal/leafnodes",
-    "internal/remote",
-    "internal/spec",
-    "internal/spec_iterator",
-    "internal/specrunner",
-    "internal/suite",
-    "internal/testingtproxy",
-    "internal/writer",
-    "reporters",
-    "reporters/stenographer",
-    "reporters/stenographer/support/go-colorable",
-    "reporters/stenographer/support/go-isatty",
-    "types",
-  ]
-  pruneopts = ""
-  revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
-  version = "v1.6.0"
-
-[[projects]]
-  digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
-  name = "github.com/onsi/gomega"
-  packages = [
-    ".",
-    "format",
-    "ghttp",
-    "internal/assertion",
-    "internal/asyncassertion",
-    "internal/oraclematcher",
-    "internal/testingtsupport",
-    "matchers",
-    "matchers/support/goraph/bipartitegraph",
-    "matchers/support/goraph/edge",
-    "matchers/support/goraph/node",
-    "matchers/support/goraph/util",
-    "types",
-  ]
-  pruneopts = ""
-  revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
-  version = "v1.4.2"
-
-[[projects]]
-  digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
-  name = "github.com/pborman/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
-  version = "v1.2"
-
-[[projects]]
-  digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
-  name = "github.com/rjeczalik/notify"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
-  version = "v0.9.1"
-
-[[projects]]
-  digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
-  name = "github.com/rs/cors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
-  version = "v1.5.0"
-
-[[projects]]
-  digest = "1:9d57e200ef5ccc4217fe0a34287308bac652435e7c6513f6263e0493d2245c56"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = ""
-  revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
-  name = "github.com/spf13/afero"
-  packages = [
-    ".",
-    "mem",
-  ]
-  pruneopts = ""
-  revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
-  version = "v1.1.2"
-
-[[projects]]
-  digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8965335b8c7107321228e3e3702cab9832751bac"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  pruneopts = ""
-  revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = ""
-  revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
-  version = "v1.0.2"
-
-[[projects]]
-  digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
-  name = "github.com/syndtr/goleveldb"
-  packages = [
-    "leveldb",
-    "leveldb/cache",
-    "leveldb/comparer",
-    "leveldb/errors",
-    "leveldb/filter",
-    "leveldb/iterator",
-    "leveldb/journal",
-    "leveldb/memdb",
-    "leveldb/opt",
-    "leveldb/storage",
-    "leveldb/table",
-    "leveldb/util",
-  ]
-  pruneopts = ""
-  revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:59b49c47c11a48f1054529207f65907c014ecf5f9a7c0d9c0f1616dec7b062ed"
-  name = "golang.org/x/crypto"
-  packages = [
-    "pbkdf2",
-    "scrypt",
-    "sha3",
-    "ssh/terminal",
-  ]
-  pruneopts = ""
-  revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
-
-[[projects]]
-  branch = "master"
-  digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "html",
-    "html/atom",
-    "html/charset",
-    "websocket",
-  ]
-  pruneopts = ""
-  revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = ""
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows",
-  ]
-  pruneopts = ""
-  revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
-
-[[projects]]
-  digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
-  name = "golang.org/x/text"
-  packages = [
-    "encoding",
-    "encoding/charmap",
-    "encoding/htmlindex",
-    "encoding/internal",
-    "encoding/internal/identifier",
-    "encoding/japanese",
-    "encoding/korean",
-    "encoding/simplifiedchinese",
-    "encoding/traditionalchinese",
-    "encoding/unicode",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "internal/utf8internal",
-    "language",
-    "runes",
-    "transform",
-    "unicode/cldr",
-    "unicode/norm",
-  ]
-  pruneopts = ""
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
-  name = "gopkg.in/fsnotify.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  source = "gopkg.in/fsnotify/fsnotify.v1"
-  version = "v1.4.7"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
-  name = "gopkg.in/natefinch/npipe.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
-
-[[projects]]
-  branch = "v1"
-  digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
-  name = "gopkg.in/tomb.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
-
-[[projects]]
-  digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/dave/jennifer/jen",
-    "github.com/ethereum/go-ethereum",
-    "github.com/ethereum/go-ethereum/accounts/abi",
-    "github.com/ethereum/go-ethereum/accounts/abi/bind",
-    "github.com/ethereum/go-ethereum/common",
-    "github.com/ethereum/go-ethereum/common/hexutil",
-    "github.com/ethereum/go-ethereum/core/rawdb",
-    "github.com/ethereum/go-ethereum/core/types",
-    "github.com/ethereum/go-ethereum/crypto",
-    "github.com/ethereum/go-ethereum/ethclient",
-    "github.com/ethereum/go-ethereum/ethdb",
-    "github.com/ethereum/go-ethereum/p2p",
-    "github.com/ethereum/go-ethereum/p2p/discv5",
-    "github.com/ethereum/go-ethereum/rlp",
-    "github.com/ethereum/go-ethereum/rpc",
-    "github.com/hashicorp/golang-lru",
-    "github.com/hpcloud/tail",
-    "github.com/jmoiron/sqlx",
-    "github.com/lib/pq",
-    "github.com/mitchellh/go-homedir",
-    "github.com/onsi/ginkgo",
-    "github.com/onsi/gomega",
-    "github.com/onsi/gomega/ghttp",
-    "github.com/sirupsen/logrus",
-    "github.com/spf13/cobra",
-    "github.com/spf13/viper",
-    "golang.org/x/net/context",
-    "golang.org/x/sync/errgroup",
-    "gopkg.in/tomb.v1",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,54 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[override]]
-  name = "gopkg.in/fsnotify.v1"
-  source = "gopkg.in/fsnotify/fsnotify.v1"
-
-[[override]]
-  name = "github.com/pressly/sup"
-  version = "0.5.3"
-
-[[constraint]]
-  name = "github.com/onsi/ginkgo"
-  version = "1.4.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jmoiron/sqlx"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/lib/pq"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "1.2.0"
-
-[[constraint]]
-  name = "github.com/spf13/cobra"
-  version = "0.0.1"
-
-[[constraint]]
-  name = "github.com/ethereum/go-ethereum"
-  revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"

Makefile

@@ -3,11 +3,6 @@ BASE = $(GOPATH)/src/$(PACKAGE)
 PKGS = go list ./... | grep -v "^vendor/"
 
 # Tools
-## Dependency management
-DEP = $(BIN)/dep
-$(BIN)/dep:
-	go get -u github.com/golang/dep/cmd/dep
-
 ## Testing library
 GINKGO = $(BIN)/ginkgo
 $(BIN)/ginkgo:
@@ -32,7 +27,7 @@ $(BIN)/gometalinter.v2:
 
 
 .PHONY: installtools
-installtools: | $(LINT) $(GOOSE) $(GINKGO) $(DEP)
+installtools: | $(LINT) $(GOOSE) $(GINKGO)
 	echo "Installing tools"
 
 .PHONY: metalint
@@ -58,11 +53,7 @@ integrationtest: | $(GINKGO) $(LINT)
 	go fmt ./...
 	$(GINKGO) -r integration_test/
 
-.PHONY: dep
+build:
-dep: | $(DEP)
-	$(DEP) ensure
-
-build: dep
 	go fmt ./...
 	go build
 

README.md

@@ -7,50 +7,56 @@
 
 
 ## Table of Contents
-1. [Background](../staging/README.md#background)
+1. [Background](#background)
-1. [Dependencies](../staging/README.md#dependencies)
+1. [Install](#install)
-1. [Install](../staging/README.md#install)
+1. [Usage](#usage)
-1. [Usage](../staging/README.md#usage)
+1. [Contributing](#contributing)
-1. [Tests](../staging/README.md#tests)
+1. [License](#license)
-1. [API](../staging/README.md#API)
-1. [Contributing](../staging/README.md#contributing)
-1. [License](../staging/README.md#license)
 
 
 ## Background
 The same data structures and encodings that make Ethereum an effective and trust-less distributed virtual machine
-complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by 
+complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by
-providing a suite of tools to ease the extraction and transformation of data into a more useful state.
+providing a suite of tools to ease the extraction and transformation of data into a more useful state, including
+allowing for exposing aggregate data from a suite of smart contracts.
 
+VulanizeDB includes processes that sync, transform and expose data. Syncing involves
+querying an Ethereum node and then persisting core data into a Postgres database. Transforming focuses on using previously synced data to
+query for and transform log event and storage data for specifically configured smart contract addresses. Exposing data is a matter of getting
+data from VulcanizeDB's underlying Postgres database and making it accessible.
 
-## Dependencies
+![VulcanizeDB Overview Diagram](documentation/diagrams/vdb-overview.png)
- - Go 1.11+
+
- - Postgres 10.6
+## Install
+
+1. [Dependencies](#dependencies)
+1. [Building the project](#building-the-project)
+1. [Setting up the database](#setting-up-the-database)
+1. [Configuring a synced Ethereum node](#configuring-a-synced-ethereum-node)
+
+### Dependencies
+ - Go 1.12+
+ - Postgres 11.2
  - Ethereum Node
    - [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.23+)
    - [Parity 1.8.11+](https://github.com/paritytech/parity/releases)
 
-
-## Install
-1. [Building the project](../staging/README.md#building-the-project)
-1. [Setting up the database](../staging/README.md#setting-up-the-database)
-1. [Configuring a synced Ethereum node](../staging/README.md#configuring-a-synced-ethereum-node)
-
 ### Building the project
 Download the codebase to your local `GOPATH` via:
 
 `go get github.com/vulcanize/vulcanizedb`
 
-Move to the project directory and use [golang/dep](https://github.com/golang/dep) to install the dependencies:
+Move to the project directory:
 
 `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
 
-`dep ensure`
+Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:
-
-Once the dependencies have been successfully installed, build the executable with:
 
 `make build`
 
+If you need to use a different dependency than what is currently defined in `go.mod`, it may helpful to look into [the replace directive](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
+This instruction enables you to point at a fork or the local filesystem for dependency resolution.
+
 If you are running into issues at this stage, ensure that `GOPATH` is defined in your shell.
 If necessary, `GOPATH` can be set in `~/.bashrc` or `~/.bash_profile`, depending upon your system.
 It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
@@ -68,7 +74,8 @@ It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
 
     * See below for configuring additional environments
     
-In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
+In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from
+localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
 
 (It should be noted that trusted auth should only be enabled on systems without sensitive data in them: development and local test databases)
 
@@ -80,8 +87,9 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
         - The IPC file is called `geth.ipc`.
         - The geth IPC file path is printed to the console when you start geth.
         - The default location is:
-          - Mac: `<full home path>/Library/Ethereum`
+          - Mac: `<full home path>/Library/Ethereum/geth.ipc`
           - Linux: `<full home path>/ethereum/geth.ipc`
+        - Note: the geth.ipc file may not exist until you've started the geth process
 
       - For Parity:
         - The IPC file is called `jsonrpc.ipc`.
@@ -98,27 +106,35 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
 
 
 ## Usage
-Usage is broken up into two processes:
+As mentioned above, VulcanizeDB's processes can be split into three categories: syncing, transforming and exposing data.
 
 ### Data syncing
-To provide data for transformations, raw Ethereum data must first be synced into vDB.
+To provide data for transformations, raw Ethereum data must first be synced into VulcanizeDB.
-This is accomplished through the use of the `headerSync`, `sync`, or `coldImport` commands.
+This is accomplished through the use of the `headerSync`, `fullSync`, or `coldImport` commands.
-These commands are described in detail [here](../staging/documentation/sync.md).
+These commands are described in detail [here](documentation/data-syncing.md).
 
 ### Data transformation
-Contract watchers use the raw data that has been synced into Postgres to filter out and apply transformations to specific data of interest.
+Data transformation uses the raw data that has been synced into Postgres to filter out and apply transformations to
+specific data of interest. Since there are different types of data that may be useful for observing smart contracts, it
+follows that there are different ways to transform this data. We've started by categorizing this into Generic and
+Custom transformers:
 
-There is a built-in `contractWatcher` command which provides generic transformation of most contract data.
+- Generic Contract Transformer: Generic contract transformation can be done using a built-in command,
-The `contractWatcher` command is described further [here](../staging/documentation/contractWatcher.md).
+`contractWatcher`, which transforms contract events provided the contract's ABI is available. It also
+provides some state variable coverage by automating polling of public methods, with some restrictions.
+`contractWatcher` is described further [here](documentation/generic-transformer.md).
 
-In many cases a custom transformer or set of transformers will need to be written to provide complete or more comprehensive coverage or to optimize other aspects of the output for a specific end-use.
+- Custom Transformers: In many cases custom transformers will need to be written to provide
-In this case we have provided the `compose`, `execute`, and `composeAndExecute` commands for running custom transformers from external repositories.
+more comprehensive coverage of contract data. In this case we have provided the `compose`, `execute`, and
+`composeAndExecute` commands for running custom transformers from external repositories. Documentation on how to write,
+build and run custom transformers as Go plugins can be found
+[here](documentation/custom-transformers.md).
 
-Usage of the `compose`, `execute`, and `composeAndExecute` commands is described further [here](../staging/documentation/composeAndExecute.md).
+### Exposing the data
+[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](documentation/postgraphile.md).
 
-Documentation on how to build custom transformers to work with these commands can be found [here](../staging/documentation/transformers.md).
 
-## Tests
+### Tests
 - Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url)
     - Note: integration tests require configuration with an archival node
 - `createdb vulcanize_private` will create the test db
@@ -126,15 +142,13 @@ Documentation on how to build custom transformers to work with these commands ca
 - `make test` will run the unit tests and skip the integration tests
 - `make integrationtest` will run just the integration tests
 
-## API
-[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](../staging/postgraphile/README.md).
-
 
 ## Contributing
-Contributions are welcome! For more on this, please see [here](../staging/documentation/contributing.md).
+Contributions are welcome!
 
-Small note: If editing the Readme, please conform to the [standard-readme specification](https://github.com/RichardLitt/standard-readme).
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
 
+For more information on contributing, please see [here](documentation/contributing.md).
 
 ## License
-[AGPL-3.0](../staging/LICENSE) © Vulcanize Inc
+[AGPL-3.0](LICENSE) © Vulcanize Inc

cmd/coldImport.go

@@ -23,10 +23,10 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/crypto"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/ethereum"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/cold_import"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/converters/cold_db"
+	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/cold_import"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/converters/cold_db"
-	vulcCommon "github.com/vulcanize/vulcanizedb/pkg/geth/converters/common"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -39,6 +39,8 @@ var coldImportCmd = &cobra.Command{
 
 Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		coldImport()
 	},
 }
@@ -55,7 +57,7 @@ func coldImport() {
 	ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
 	ethDB, err := ethereum.CreateDatabase(ethDBConfig)
 	if err != nil {
-		log.Fatal("Error connecting to ethereum db: ", err)
+		LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
 	}
 	mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
 	if syncAll {
@@ -63,10 +65,10 @@ func coldImport() {
 		endingBlockNumber = mostRecentBlockNumberInDb
 	}
 	if endingBlockNumber < startingBlockNumber {
-		log.Fatal("Ending block number must be greater than starting block number for cold import.")
+		LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
 	}
 	if endingBlockNumber > mostRecentBlockNumberInDb {
-		log.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
+		LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
 	}
 
 	// init pg db
@@ -76,13 +78,13 @@ func coldImport() {
 	nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
 	coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
 	if err != nil {
-		log.Fatal("Error getting node: ", err)
+		LogWithCommand.Fatal("Error getting node: ", err)
 	}
 	pgDB := utils.LoadPostgres(databaseConfig, coldNode)
 
 	// init cold importer deps
 	blockRepository := repositories.NewBlockRepository(&pgDB)
-	receiptRepository := repositories.ReceiptRepository{DB: &pgDB}
+	receiptRepository := repositories.FullSyncReceiptRepository{DB: &pgDB}
 	transactionConverter := cold_db.NewColdDbTransactionConverter()
 	blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
 
@@ -90,6 +92,6 @@ func coldImport() {
 	coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
 	err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
 	if err != nil {
-		log.Fatal("Error executing cold import: ", err)
+		LogWithCommand.Fatal("Error executing cold import: ", err)
 	}
 }

cmd/compose.go

@@ -1,17 +1,18 @@
-// Copyright © 2019 Vulcanize, Inc
+// VulcanizeDB
-//
+// Copyright © 2019 Vulcanize
+
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
-//
+
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
-//
+
 // You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package cmd
 
@@ -101,6 +102,8 @@ single config file or in separate command instances using different config files
 Specify config location when executing the command:
 ./vulcanizedb compose --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		compose()
 	},
 }
@@ -110,23 +113,25 @@ func compose() {
 	prepConfig()
 
 	// Generate code to build the plugin according to the config file
-	log.Info("generating plugin")
+	LogWithCommand.Info("generating plugin")
 	generator, err := p2.NewGenerator(genConfig, databaseConfig)
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Debug("initializing plugin generator failed")
+		LogWithCommand.Fatal(err)
 	}
 	err = generator.GenerateExporterPlugin()
 	if err != nil {
-		log.Debug("generating plugin failed")
+		LogWithCommand.Debug("generating plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Debug("getting plugin path failed")
+		LogWithCommand.Fatal(err)
 	}
 	fmt.Printf("Composed plugin %s", pluginPath)
-	log.Info("plugin .so file output to", pluginPath)
+	LogWithCommand.Info("plugin .so file output to ", pluginPath)
 }
 
 func init() {
@@ -134,38 +139,38 @@ func init() {
 }
 
 func prepConfig() {
-	log.Info("configuring plugin")
+	LogWithCommand.Info("configuring plugin")
 	names := viper.GetStringSlice("exporter.transformerNames")
 	transformers := make(map[string]config.Transformer)
 	for _, name := range names {
 		transformer := viper.GetStringMapString("exporter." + name)
 		p, pOK := transformer["path"]
 		if !pOK || p == "" {
-			log.Fatal(name, "transformer config is missing `path` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `path` value")
 		}
 		r, rOK := transformer["repository"]
 		if !rOK || r == "" {
-			log.Fatal(name, "transformer config is missing `repository` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
 		}
 		m, mOK := transformer["migrations"]
 		if !mOK || m == "" {
-			log.Fatal(name, "transformer config is missing `migrations` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
 		}
 		mr, mrOK := transformer["rank"]
 		if !mrOK || mr == "" {
-			log.Fatal(name, "transformer config is missing `rank` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
 		}
 		rank, err := strconv.ParseUint(mr, 10, 64)
 		if err != nil {
-			log.Fatal(name, "migration `rank` can't be converted to an unsigned integer")
+			LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
 		}
 		t, tOK := transformer["type"]
 		if !tOK {
-			log.Fatal(name, "transformer config is missing `type` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `type` value")
 		}
 		transformerType := config.GetTransformerType(t)
 		if transformerType == config.UnknownTransformerType {
-			log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
+			LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
 		}
 
 		transformers[name] = config.Transformer{

cmd/composeAndExecute.go

@@ -1,21 +1,26 @@
-// Copyright © 2019 Vulcanize, Inc
+// VulcanizeDB
-//
+// Copyright © 2019 Vulcanize
+
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
-//
+
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
-//
+
 // You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package cmd
 
 import (
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"os"
 	"plugin"
 	syn "sync"
@@ -24,9 +29,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
 	"github.com/vulcanize/vulcanizedb/utils"
@@ -105,6 +108,8 @@ single config file or in separate command instances using different config files
 Specify config location when executing the command:
 ./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		composeAndExecute()
 	},
 }
@@ -114,44 +119,44 @@ func composeAndExecute() {
 	prepConfig()
 
 	// Generate code to build the plugin according to the config file
-	log.Info("generating plugin")
+	LogWithCommand.Info("generating plugin")
 	generator, err := p2.NewGenerator(genConfig, databaseConfig)
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	err = generator.GenerateExporterPlugin()
 	if err != nil {
-		log.Debug("generating plugin failed")
+		LogWithCommand.Debug("generating plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Get the plugin path and load the plugin
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	if !genConfig.Save {
 		defer helpers.ClearFiles(pluginPath)
 	}
-	log.Info("linking plugin", pluginPath)
+	LogWithCommand.Info("linking plugin ", pluginPath)
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
-		log.Debug("linking plugin failed")
+		LogWithCommand.Debug("linking plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Load the `Exporter` symbol from the plugin
-	log.Info("loading transformers from plugin")
+	LogWithCommand.Info("loading transformers from plugin")
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
-		log.Debug("loading Exporter symbol failed")
+		LogWithCommand.Debug("loading Exporter symbol failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
-		log.Debug("plugged-in symbol not of type Exporter")
+		LogWithCommand.Debug("plugged-in symbol not of type Exporter")
 		os.Exit(1)
 	}
 
@@ -167,18 +172,35 @@ func composeAndExecute() {
 	var wg syn.WaitGroup
 	if len(ethEventInitializers) > 0 {
 		ew := watcher.NewEventWatcher(&db, blockChain)
-		ew.AddTransformers(ethEventInitializers)
+		err := ew.AddTransformers(ethEventInitializers)
+		if err != nil {
+			LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
+		}
 		wg.Add(1)
 		go watchEthEvents(&ew, &wg)
 	}
 
 	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
+		switch storageDiffsSource {
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+		case "geth":
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			log.Debug("fetching storage diffs from geth pub sub")
-		sw.AddTransformers(ethStorageInitializers)
+			rpcClient, _ := getClients()
-		wg.Add(1)
+			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
-		go watchEthStorage(&sw, &wg)
+			payloadChan := make(chan statediff.Payload)
+			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		default:
+			log.Debug("fetching storage diffs from csv")
+			tailer := fs.FileTailer{Path: storageDiffsPath}
+			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		}
 	}
 
 	if len(ethContractInitializers) > 0 {
@@ -193,5 +215,5 @@ func composeAndExecute() {
 func init() {
 	rootCmd.AddCommand(composeAndExecuteCmd)
 	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
+	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
 }

cmd/contractWatcher.go

@@ -26,7 +26,7 @@ import (
 
 	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
-	lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
+	ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -79,6 +79,8 @@ Requires a .toml config file:
         piping = true
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		contractWatcher()
 	},
 }
@@ -99,22 +101,22 @@ func contractWatcher() {
 	con.PrepConfig()
 	switch mode {
 	case "header":
-		t = lt.NewTransformer(con, blockChain, &db)
+		t = ht.NewTransformer(con, blockChain, &db)
 	case "full":
 		t = ft.NewTransformer(con, blockChain, &db)
 	default:
-		log.Fatal("Invalid mode")
+		LogWithCommand.Fatal("Invalid mode")
 	}
 
 	err := t.Init()
 	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
+		LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
 	}
 
 	for range ticker.C {
 		err = t.Execute()
 		if err != nil {
-			log.Error("Execution error for transformer:", t.GetConfig().Name, err)
+			LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
 		}
 	}
 }

cmd/execute.go

@@ -1,23 +1,27 @@
-// Copyright © 2019 Vulcanize, Inc
+// VulcanizeDB
-//
+// Copyright © 2019 Vulcanize
+
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
-//
+
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
-//
+
 // You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package cmd
 
 import (
 	"fmt"
-	"os"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"plugin"
 	syn "sync"
 	"time"
@@ -26,11 +30,9 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -64,6 +66,8 @@ must have been composed by the same version of vulcanizedb or else it will not b
 Specify config location when executing the command:
 ./vulcanizedb execute --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		execute()
 	},
 }
@@ -75,30 +79,29 @@ func execute() {
 	// Get the plugin path and load the plugin
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	fmt.Printf("Executing plugin %s", pluginPath)
-	log.Info("linking plugin", pluginPath)
+	LogWithCommand.Info("linking plugin ", pluginPath)
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
-		log.Debug("linking plugin failed")
+		LogWithCommand.Warn("linking plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Load the `Exporter` symbol from the plugin
-	log.Info("loading transformers from plugin")
+	LogWithCommand.Info("loading transformers from plugin")
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
-		log.Debug("loading Exporter symbol failed")
+		LogWithCommand.Warn("loading Exporter symbol failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
-		log.Debug("plugged-in symbol not of type Exporter")
+		LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
-		os.Exit(1)
 	}
 
 	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
@@ -113,18 +116,35 @@ func execute() {
 	var wg syn.WaitGroup
 	if len(ethEventInitializers) > 0 {
 		ew := watcher.NewEventWatcher(&db, blockChain)
-		ew.AddTransformers(ethEventInitializers)
+		err = ew.AddTransformers(ethEventInitializers)
+		if err != nil {
+			LogWithCommand.Fatalf("failed to add event transformer initializers to watcher: %s", err.Error())
+		}
 		wg.Add(1)
 		go watchEthEvents(&ew, &wg)
 	}
 
 	if len(ethStorageInitializers) > 0 {
-		tailer := fs.FileTailer{Path: storageDiffsPath}
+		switch storageDiffsSource {
-		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+		case "geth":
-		sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			log.Debug("fetching storage diffs from geth pub sub")
-		sw.AddTransformers(ethStorageInitializers)
+			rpcClient, _ := getClients()
-		wg.Add(1)
+			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
-		go watchEthStorage(&sw, &wg)
+			payloadChan := make(chan statediff.Payload)
+			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		default:
+			log.Debug("fetching storage diffs from csv")
+			tailer := fs.FileTailer{Path: storageDiffsPath}
+			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
+			sw.AddTransformers(ethStorageInitializers)
+			wg.Add(1)
+			go watchEthStorage(&sw, &wg)
+		}
 	}
 
 	if len(ethContractInitializers) > 0 {
@@ -139,7 +159,7 @@ func execute() {
 func init() {
 	rootCmd.AddCommand(executeCmd)
 	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
+	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
 }
 
 type Exporter interface {
@@ -149,37 +169,36 @@ type Exporter interface {
 func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the EventTransformerInitializer set using the watcher
-	log.Info("executing event transformers")
+	LogWithCommand.Info("executing event transformers")
 	var recheck constants.TransformerExecution
 	if recheckHeadersArg {
 		recheck = constants.HeaderRecheck
 	} else {
-		recheck = constants.HeaderMissing
+		recheck = constants.HeaderUnchecked
 	}
-	ticker := time.NewTicker(pollingInterval)
+	err := w.Execute(recheck)
-	defer ticker.Stop()
+	if err != nil {
-	for range ticker.C {
+		LogWithCommand.Fatalf("error executing event watcher: %s", err.Error())
-		w.Execute(recheck)
 	}
 }
 
-func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
+func watchEthStorage(w watcher.IStorageWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the StorageTransformerInitializer set using the storage watcher
-	log.Info("executing storage transformers")
+	LogWithCommand.Info("executing storage transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {
 		errs := make(chan error)
-		rows := make(chan storageUtils.StorageDiffRow)
+		diffs := make(chan storageUtils.StorageDiff)
-		w.Execute(rows, errs, queueRecheckInterval)
+		w.Execute(diffs, errs, queueRecheckInterval)
 	}
 }
 
 func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the ContractTransformerInitializer set using the contract watcher
-	log.Info("executing contract_watcher transformers")
+	LogWithCommand.Info("executing contract_watcher transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {

cmd/fullSync.go

@@ -29,14 +29,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
-// syncCmd represents the sync command
+// fullSyncCmd represents the fullSync command
-var syncCmd = &cobra.Command{
+var fullSyncCmd = &cobra.Command{
-	Use:   "sync",
+	Use:   "fullSync",
 	Short: "Syncs VulcanizeDB with local ethereum node",
 	Long: `Syncs VulcanizeDB with local ethereum node. Populates
 Postgres with blocks, transactions, receipts, and logs.
 
-./vulcanizedb sync --starting-block-number 0 --config public.toml
+./vulcanizedb fullSync --starting-block-number 0 --config public.toml
 
 Expects ethereum node to be running and requires a .toml config:
 
@@ -49,38 +49,39 @@ Expects ethereum node to be running and requires a .toml config:
   ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		sync()
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
+		fullSync()
 	},
 }
 
 func init() {
-	rootCmd.AddCommand(syncCmd)
+	rootCmd.AddCommand(fullSyncCmd)
-
+	fullSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
-	syncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
 }
 
 func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
 	populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
 	if err != nil {
-		log.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
+		LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
 	}
 	missingBlocksPopulated <- populated
 }
 
-func sync() {
+func fullSync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 
 	blockChain := getBlockChain()
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
-		log.Error("sync: Error getting last block: ", err)
+		LogWithCommand.Error("fullSync: Error getting last block: ", err)
 	}
 	if lastBlock.Int64() == 0 {
-		log.Fatal("geth initial: state sync not finished")
+		LogWithCommand.Fatal("geth initial: state sync not finished")
 	}
 	if startingBlockNumber > lastBlock.Int64() {
-		log.Fatal("sync: starting block number > current block number")
+		LogWithCommand.Fatal("fullSync: starting block number > current block number")
 	}
 
 	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
@@ -94,9 +95,9 @@ func sync() {
 		case <-ticker.C:
 			window, err := validator.ValidateBlocks()
 			if err != nil {
-				log.Error("sync: error in validateBlocks: ", err)
+				LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
 			}
-			log.Info(window.GetString())
+			LogWithCommand.Debug(window.GetString())
 		case <-missingBlocksPopulated:
 			go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
 		}

cmd/headerSync.go

@@ -25,7 +25,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
@@ -50,6 +50,8 @@ Expects ethereum node to be running and requires a .toml config:
   ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		headerSync()
 	},
 }
@@ -64,7 +66,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
 	if err != nil {
 		// TODO Lots of possible errors in the call stack above. If errors occur, we still put
 		// 0 in the channel, triggering another round
-		log.Error("backfillAllHeaders: Error populating headers: ", err)
+		LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
 	}
 	missingBlocksPopulated <- populated
 }
@@ -86,9 +88,9 @@ func headerSync() {
 		case <-ticker.C:
 			window, err := validator.ValidateHeaders()
 			if err != nil {
-				log.Error("headerSync: ValidateHeaders failed: ", err)
+				LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
 			}
-			log.Info(window.GetString())
+			LogWithCommand.Debug(window.GetString())
 		case n := <-missingBlocksPopulated:
 			if n == 0 {
 				time.Sleep(3 * time.Second)
@@ -98,15 +100,15 @@ func headerSync() {
 	}
 }
 
-func validateArgs(blockChain *geth.BlockChain) {
+func validateArgs(blockChain *eth.BlockChain) {
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
-		log.Error("validateArgs: Error getting last block: ", err)
+		LogWithCommand.Error("validateArgs: Error getting last block: ", err)
 	}
 	if lastBlock.Int64() == 0 {
-		log.Fatal("geth initial: state sync not finished")
+		LogWithCommand.Fatal("geth initial: state sync not finished")
 	}
 	if startingBlockNumber > lastBlock.Int64() {
-		log.Fatal("starting block number > current block number")
+		LogWithCommand.Fatal("starting block number > current block number")
 	}
 }

cmd/root.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -29,10 +28,10 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 )
 
 var (
@@ -47,6 +46,9 @@ var (
 	syncAll              bool
 	endingBlockNumber    int64
 	recheckHeadersArg    bool
+	SubCommand           string
+	LogWithCommand       log.Entry
+	storageDiffsSource   string
 )
 
 const (
@@ -56,21 +58,30 @@ const (
 
 var rootCmd = &cobra.Command{
 	Use:              "vulcanizedb",
-	PersistentPreRun: database,
+	PersistentPreRun: initFuncs,
 }
 
 func Execute() {
 	log.Info("----- Starting vDB -----")
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
-		os.Exit(1)
 	}
 }
 
-func database(cmd *cobra.Command, args []string) {
+func initFuncs(cmd *cobra.Command, args []string) {
+	setViperConfigs()
+	logLvlErr := logLevel()
+	if logLvlErr != nil {
+		log.Fatal("Could not set log level: ", logLvlErr)
+	}
+
+}
+
+func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
 	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
+	storageDiffsSource = viper.GetString("storageDiffs.source")
 	databaseConfig = config.Database{
 		Name:     viper.GetString("database.name"),
 		Hostname: viper.GetString("database.hostname"),
@@ -81,6 +92,19 @@ func database(cmd *cobra.Command, args []string) {
 	viper.Set("database.config", databaseConfig)
 }
 
+func logLevel() error {
+	lvl, err := log.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	log.SetLevel(lvl)
+	if lvl > log.InfoLevel {
+		log.SetReportCaller(true)
+	}
+	log.Info("Log level set to ", lvl.String())
+	return nil
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 	// When searching for env variables, replace dots in config keys with underscores
@@ -96,7 +120,9 @@ func init() {
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
+	rootCmd.PersistentFlags().String("storageDiffs-source", "csv", "where to get the state diffs: csv or geth")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -106,7 +132,9 @@ func init() {
 	viper.BindPFlag("client.ipcPath", rootCmd.PersistentFlags().Lookup("client-ipcPath"))
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
+	viper.BindPFlag("storageDiffs.source", rootCmd.PersistentFlags().Lookup("storageDiffs-source"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
 
 func initConfig() {
@@ -116,29 +144,33 @@ func initConfig() {
 		noConfigError := "No config file passed with --config flag"
 		fmt.Println("Error: ", noConfigError)
 		log.Fatal(noConfigError)
-		os.Exit(1)
 	}
 
 	if err := viper.ReadInConfig(); err == nil {
 		log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
 	} else {
 		invalidConfigError := "Couldn't read config file"
-		fmt.Println("Error: ", invalidConfigError)
+		formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error())
-		log.Fatal(invalidConfigError)
+		log.Fatal(formattedError)
-		os.Exit(1)
 	}
 }
 
-func getBlockChain() *geth.BlockChain {
+func getBlockChain() *eth.BlockChain {
-	rawRpcClient, err := rpc.Dial(ipc)
+	rpcClient, ethClient := getClients()
-
-	if err != nil {
-		log.Fatal(err)
-	}
-	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
-	ethClient := ethclient.NewClient(rawRpcClient)
 	vdbEthClient := client.NewEthClient(ethClient)
 	vdbNode := node.MakeNode(rpcClient)
 	transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-	return geth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+	return eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+}
+
+func getClients() (client.RpcClient, *ethclient.Client) {
+	rawRpcClient, err := rpc.Dial(ipc)
+
+	if err != nil {
+		LogWithCommand.Fatal(err)
+	}
+	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
+	ethClient := ethclient.NewClient(rawRpcClient)
+
+	return rpcClient, ethClient
 }

db/migrations/00007_create_full_sync_logs_table.sql

@@ -0,0 +1,18 @@
+-- +goose Up
+CREATE TABLE full_sync_logs
+(
+    id           SERIAL PRIMARY KEY,
+    block_number BIGINT,
+    address      VARCHAR(66),
+    tx_hash      VARCHAR(66),
+    index        BIGINT,
+    topic0       VARCHAR(66),
+    topic1       VARCHAR(66),
+    topic2       VARCHAR(66),
+    topic3       VARCHAR(66),
+    data         TEXT
+);
+
+
+-- +goose Down
+DROP TABLE full_sync_logs;

db/migrations/00007_create_logs_table.sql

@@ -1,19 +0,0 @@
--- +goose Up
-CREATE TABLE logs (
-  id           SERIAL PRIMARY KEY,
-  block_number BIGINT,
-  address      VARCHAR(66),
-  tx_hash      VARCHAR(66),
-  index        BIGINT,
-  topic0       VARCHAR(66),
-  topic1       VARCHAR(66),
-  topic2       VARCHAR(66),
-  topic3       VARCHAR(66),
-  data         TEXT,
-  CONSTRAINT log_uc UNIQUE (block_number, index)
-);
-
-
-
--- +goose Down
-DROP TABLE logs;

db/migrations/00013_add_address_table.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+CREATE TABLE public.addresses
+(
+    id      SERIAL PRIMARY KEY,
+    address character varying(42),
+    UNIQUE (address)
+);
+
+-- +goose Down
+DROP TABLE public.addresses;

db/migrations/00013_create_receipts_table.sql

@@ -1,16 +0,0 @@
--- +goose Up
-CREATE TABLE full_sync_receipts
-(
-  id                  SERIAL PRIMARY KEY,
-  transaction_id      INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
-  contract_address    VARCHAR(42),
-  cumulative_gas_used NUMERIC,
-  gas_used            NUMERIC,
-  state_root          VARCHAR(66),
-  status              INTEGER,
-  tx_hash             VARCHAR(66)
-);
-
-
--- +goose Down
-DROP TABLE full_sync_receipts;

db/migrations/00014_create_receipts_table.sql

@@ -0,0 +1,16 @@
+-- +goose Up
+CREATE TABLE full_sync_receipts
+(
+    id                  SERIAL PRIMARY KEY,
+    transaction_id      INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
+    contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
+    cumulative_gas_used NUMERIC,
+    gas_used            NUMERIC,
+    state_root          VARCHAR(66),
+    status              INTEGER,
+    tx_hash             VARCHAR(66)
+);
+
+
+-- +goose Down
+DROP TABLE full_sync_receipts;

db/migrations/00015_add_receipts_fk_to_logs.sql

@@ -1,23 +0,0 @@
--- +goose Up
-ALTER TABLE logs
-    DROP CONSTRAINT log_uc;
-
-ALTER TABLE logs
-  ADD COLUMN receipt_id INT;
-
-ALTER TABLE logs
-  ADD CONSTRAINT receipts_fk
-FOREIGN KEY (receipt_id)
-REFERENCES full_sync_receipts (id)
-ON DELETE CASCADE;
-
-
--- +goose Down
-ALTER TABLE logs
-  DROP CONSTRAINT receipts_fk;
-
-ALTER TABLE logs
-  DROP COLUMN receipt_id;
-
-ALTER TABLE logs
-  ADD CONSTRAINT log_uc UNIQUE (block_number, index);

db/migrations/00016_add_receipts_fk_to_logs.sql

@@ -0,0 +1,17 @@
+-- +goose Up
+ALTER TABLE full_sync_logs
+    ADD COLUMN receipt_id INT;
+
+ALTER TABLE full_sync_logs
+    ADD CONSTRAINT receipts_fk
+        FOREIGN KEY (receipt_id)
+            REFERENCES full_sync_receipts (id)
+            ON DELETE CASCADE;
+
+
+-- +goose Down
+ALTER TABLE full_sync_logs
+    DROP CONSTRAINT receipts_fk;
+
+ALTER TABLE full_sync_logs
+    DROP COLUMN receipt_id;

db/migrations/00017_create_watched_event_logs.sql

@@ -1,34 +0,0 @@
--- +goose Up
-CREATE VIEW block_stats AS
-  SELECT
-    max(block_number) AS max_block,
-    min(block_number) AS min_block
-  FROM logs;
-
-CREATE VIEW watched_event_logs AS
-  SELECT
-    log_filters.name,
-    logs.id,
-    block_number,
-    logs.address,
-    tx_hash,
-    index,
-    logs.topic0,
-    logs.topic1,
-    logs.topic2,
-    logs.topic3,
-    data,
-    receipt_id
-  FROM log_filters
-    CROSS JOIN block_stats
-    JOIN logs ON logs.address = log_filters.address
-                 AND logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
-                 AND logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
-  WHERE (log_filters.topic0 = logs.topic0 OR log_filters.topic0 ISNULL)
-        AND (log_filters.topic1 = logs.topic1 OR log_filters.topic1 ISNULL)
-        AND (log_filters.topic2 = logs.topic2 OR log_filters.topic2 ISNULL)
-        AND (log_filters.topic3 = logs.topic3 OR log_filters.topic3 ISNULL);
-
--- +goose Down
-DROP VIEW watched_event_logs;
-DROP VIEW block_stats;

db/migrations/00018_create_watched_event_logs.sql

@@ -0,0 +1,32 @@
+-- +goose Up
+CREATE VIEW block_stats AS
+SELECT max(block_number) AS max_block,
+       min(block_number) AS min_block
+FROM full_sync_logs;
+
+CREATE VIEW watched_event_logs AS
+SELECT log_filters.name,
+       full_sync_logs.id,
+       block_number,
+       full_sync_logs.address,
+       tx_hash,
+       index,
+       full_sync_logs.topic0,
+       full_sync_logs.topic1,
+       full_sync_logs.topic2,
+       full_sync_logs.topic3,
+       data,
+       receipt_id
+FROM log_filters
+         CROSS JOIN block_stats
+         JOIN full_sync_logs ON full_sync_logs.address = log_filters.address
+    AND full_sync_logs.block_number >= coalesce(log_filters.from_block, block_stats.min_block)
+    AND full_sync_logs.block_number <= coalesce(log_filters.to_block, block_stats.max_block)
+WHERE (log_filters.topic0 = full_sync_logs.topic0 OR log_filters.topic0 ISNULL)
+  AND (log_filters.topic1 = full_sync_logs.topic1 OR log_filters.topic1 ISNULL)
+  AND (log_filters.topic2 = full_sync_logs.topic2 OR log_filters.topic2 ISNULL)
+  AND (log_filters.topic3 = full_sync_logs.topic3 OR log_filters.topic3 ISNULL);
+
+-- +goose Down
+DROP VIEW watched_event_logs;
+DROP VIEW block_stats;

db/migrations/00022_create_headers_table.sql

@@ -1,17 +0,0 @@
--- +goose Up
-CREATE TABLE public.headers (
-  id                    SERIAL PRIMARY KEY,
-  hash                  VARCHAR(66),
-  block_number          BIGINT,
-  raw                   JSONB,
-  block_timestamp       NUMERIC,
-  eth_node_id           INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
-  eth_node_fingerprint  VARCHAR(128)
-);
-
--- Index is removed when table is
-CREATE INDEX headers_block_number ON public.headers (block_number);
-
-
--- +goose Down
-DROP TABLE public.headers;

db/migrations/00023_create_headers_table.sql

@@ -0,0 +1,20 @@
+-- +goose Up
+CREATE TABLE public.headers
+(
+    id                   SERIAL PRIMARY KEY,
+    hash                 VARCHAR(66),
+    block_number         BIGINT,
+    raw                  JSONB,
+    block_timestamp      NUMERIC,
+    check_count          INTEGER NOT NULL DEFAULT 0,
+    eth_node_id          INTEGER NOT NULL REFERENCES eth_nodes (id) ON DELETE CASCADE,
+    eth_node_fingerprint VARCHAR(128),
+    UNIQUE (block_number, hash, eth_node_fingerprint)
+);
+
+-- Index is removed when table is
+CREATE INDEX headers_block_number ON public.headers (block_number);
+
+
+-- +goose Down
+DROP TABLE public.headers;

db/migrations/00025_create_queued_storage.sql

@@ -5,7 +5,8 @@ CREATE TABLE public.queued_storage (
   block_hash    BYTEA,
   contract      BYTEA,
   storage_key   BYTEA,
-  storage_value BYTEA
+  storage_value BYTEA,
+  UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
 );
 
 -- +goose Down

db/migrations/00026_create_header_sync_receipts_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE header_sync_receipts(
-  id                  SERIAL PRIMARY KEY,
-  transaction_id      INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE,
-  header_id           INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
-  contract_address    VARCHAR(42),
-  cumulative_gas_used NUMERIC,
-  gas_used            NUMERIC,
-  state_root          VARCHAR(66),
-  status              INTEGER,
-  tx_hash             VARCHAR(66),
-  rlp                 BYTEA,
-  UNIQUE(header_id, transaction_id)
-);
-
-
--- +goose Down
-DROP TABLE header_sync_receipts;

db/migrations/00027_create_header_sync_receipts_table.sql

@@ -0,0 +1,19 @@
+-- +goose Up
+CREATE TABLE header_sync_receipts
+(
+    id                  SERIAL PRIMARY KEY,
+    transaction_id      INTEGER NOT NULL REFERENCES header_sync_transactions (id) ON DELETE CASCADE,
+    header_id           INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+    contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
+    cumulative_gas_used NUMERIC,
+    gas_used            NUMERIC,
+    state_root          VARCHAR(66),
+    status              INTEGER,
+    tx_hash             VARCHAR(66),
+    rlp                 BYTEA,
+    UNIQUE (header_id, transaction_id)
+);
+
+
+-- +goose Down
+DROP TABLE header_sync_receipts;

db/migrations/00029_create_header_sync_logs_table.sql

@@ -0,0 +1,22 @@
+-- +goose Up
+-- SQL in this section is executed when the migration is applied.
+CREATE TABLE header_sync_logs
+(
+    id           SERIAL PRIMARY KEY,
+    header_id    INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+    address      INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
+    topics       BYTEA[],
+    data         BYTEA,
+    block_number BIGINT,
+    block_hash   VARCHAR(66),
+    tx_hash      VARCHAR(66),
+    tx_index     INTEGER,
+    log_index    INTEGER,
+    raw          JSONB,
+    transformed  BOOL    NOT NULL DEFAULT FALSE,
+    UNIQUE (header_id, tx_index, log_index)
+);
+
+-- +goose Down
+-- SQL in this section is executed when the migration is rolled back.
+DROP TABLE header_sync_logs;

db/migrations/00030_create_watched_logs_table.sql

@@ -0,0 +1,12 @@
+-- +goose Up
+-- SQL in this section is executed when the migration is applied.
+CREATE TABLE public.watched_logs
+(
+    id               SERIAL PRIMARY KEY,
+    contract_address VARCHAR(42),
+    topic_zero       VARCHAR(66)
+);
+
+-- +goose Down
+-- SQL in this section is executed when the migration is rolled back.
+DROP TABLE public.watched_logs;

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.6
+-- Dumped from database version 11.4
--- Dumped by pg_dump version 10.6
+-- Dumped by pg_dump version 11.4
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -12,32 +12,49 @@ SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
 SELECT pg_catalog.set_config('search_path', '', false);
 SET check_function_bodies = false;
+SET xmloption = content;
 SET client_min_messages = warning;
 SET row_security = off;
 
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
---
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
 SET default_tablespace = '';
 
 SET default_with_oids = false;
 
 --
--- Name: logs; Type: TABLE; Schema: public; Owner: -
+-- Name: addresses; Type: TABLE; Schema: public; Owner: -
 --
 
-CREATE TABLE public.logs (
+CREATE TABLE public.addresses (
+    id integer NOT NULL,
+    address character varying(42)
+);
+
+
+--
+-- Name: addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.addresses_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id;
+
+
+--
+-- Name: full_sync_logs; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.full_sync_logs (
     id integer NOT NULL,
     block_number bigint,
     address character varying(66),
@@ -57,9 +74,9 @@ CREATE TABLE public.logs (
 --
 
 CREATE VIEW public.block_stats AS
- SELECT max(logs.block_number) AS max_block,
+ SELECT max(full_sync_logs.block_number) AS max_block,
-    min(logs.block_number) AS min_block
+    min(full_sync_logs.block_number) AS min_block
-   FROM public.logs;
+   FROM public.full_sync_logs;
 
 
 --
@@ -151,13 +168,33 @@ CREATE TABLE public.eth_nodes (
 );
 
 
+--
+-- Name: full_sync_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.full_sync_logs_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: full_sync_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.full_sync_logs_id_seq OWNED BY public.full_sync_logs.id;
+
+
 --
 -- Name: full_sync_receipts; Type: TABLE; Schema: public; Owner: -
 --
 
 CREATE TABLE public.full_sync_receipts (
     id integer NOT NULL,
-    contract_address character varying(42),
+    contract_address_id integer NOT NULL,
     cumulative_gas_used numeric,
     gas_used numeric,
     state_root character varying(66),
@@ -259,6 +296,46 @@ CREATE SEQUENCE public.goose_db_version_id_seq
 ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;
 
 
+--
+-- Name: header_sync_logs; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.header_sync_logs (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    address integer NOT NULL,
+    topics bytea[],
+    data bytea,
+    block_number bigint,
+    block_hash character varying(66),
+    tx_hash character varying(66),
+    tx_index integer,
+    log_index integer,
+    raw jsonb,
+    transformed boolean DEFAULT false NOT NULL
+);
+
+
+--
+-- Name: header_sync_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.header_sync_logs_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: header_sync_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.header_sync_logs_id_seq OWNED BY public.header_sync_logs.id;
+
+
 --
 -- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: -
 --
@@ -267,7 +344,7 @@ CREATE TABLE public.header_sync_receipts (
     id integer NOT NULL,
     transaction_id integer NOT NULL,
     header_id integer NOT NULL,
-    contract_address character varying(42),
+    contract_address_id integer NOT NULL,
     cumulative_gas_used numeric,
     gas_used numeric,
     state_root character varying(66),
@@ -347,6 +424,7 @@ CREATE TABLE public.headers (
     block_number bigint,
     raw jsonb,
     block_timestamp numeric,
+    check_count integer DEFAULT 0 NOT NULL,
     eth_node_id integer NOT NULL,
     eth_node_fingerprint character varying(128)
 );
@@ -412,26 +490,6 @@ CREATE SEQUENCE public.log_filters_id_seq
 ALTER SEQUENCE public.log_filters_id_seq OWNED BY public.log_filters.id;
 
 
---
--- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.logs_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.logs_id_seq OWNED BY public.logs.id;
-
-
 --
 -- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -560,21 +618,59 @@ ALTER SEQUENCE public.watched_contracts_contract_id_seq OWNED BY public.watched_
 
 CREATE VIEW public.watched_event_logs AS
  SELECT log_filters.name,
-    logs.id,
+    full_sync_logs.id,
-    logs.block_number,
+    full_sync_logs.block_number,
-    logs.address,
+    full_sync_logs.address,
-    logs.tx_hash,
+    full_sync_logs.tx_hash,
-    logs.index,
+    full_sync_logs.index,
-    logs.topic0,
+    full_sync_logs.topic0,
-    logs.topic1,
+    full_sync_logs.topic1,
-    logs.topic2,
+    full_sync_logs.topic2,
-    logs.topic3,
+    full_sync_logs.topic3,
-    logs.data,
+    full_sync_logs.data,
-    logs.receipt_id
+    full_sync_logs.receipt_id
    FROM ((public.log_filters
      CROSS JOIN public.block_stats)
-     JOIN public.logs ON ((((logs.address)::text = (log_filters.address)::text) AND (logs.block_number >= COALESCE(log_filters.from_block, block_stats.min_block)) AND (logs.block_number <= COALESCE(log_filters.to_block, block_stats.max_block)))))
+     JOIN public.full_sync_logs ON ((((full_sync_logs.address)::text = (log_filters.address)::text) AND (full_sync_logs.block_number >= COALESCE(log_filters.from_block, block_stats.min_block)) AND (full_sync_logs.block_number <= COALESCE(log_filters.to_block, block_stats.max_block)))))
-  WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
+  WHERE ((((log_filters.topic0)::text = (full_sync_logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (full_sync_logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (full_sync_logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (full_sync_logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
+
+
+--
+-- Name: watched_logs; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.watched_logs (
+    id integer NOT NULL,
+    contract_address character varying(42),
+    topic_zero character varying(66)
+);
+
+
+--
+-- Name: watched_logs_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.watched_logs_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: watched_logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.watched_logs_id_seq OWNED BY public.watched_logs.id;
+
+
+--
+-- Name: addresses id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass);
 
 
 --
@@ -598,6 +694,13 @@ ALTER TABLE ONLY public.checked_headers ALTER COLUMN id SET DEFAULT nextval('pub
 ALTER TABLE ONLY public.eth_nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
 
 
+--
+-- Name: full_sync_logs id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.full_sync_logs ALTER COLUMN id SET DEFAULT nextval('public.full_sync_logs_id_seq'::regclass);
+
+
 --
 -- Name: full_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -619,6 +722,13 @@ ALTER TABLE ONLY public.full_sync_transactions ALTER COLUMN id SET DEFAULT nextv
 ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);
 
 
+--
+-- Name: header_sync_logs id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_logs ALTER COLUMN id SET DEFAULT nextval('public.header_sync_logs_id_seq'::regclass);
+
+
 --
 -- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -647,13 +757,6 @@ ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.head
 ALTER TABLE ONLY public.log_filters ALTER COLUMN id SET DEFAULT nextval('public.log_filters_id_seq'::regclass);
 
 
---
--- Name: logs id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.logs ALTER COLUMN id SET DEFAULT nextval('public.logs_id_seq'::regclass);
-
-
 --
 -- Name: queued_storage id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -675,6 +778,29 @@ ALTER TABLE ONLY public.uncles ALTER COLUMN id SET DEFAULT nextval('public.uncle
 ALTER TABLE ONLY public.watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('public.watched_contracts_contract_id_seq'::regclass);
 
 
+--
+-- Name: watched_logs id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.watched_logs ALTER COLUMN id SET DEFAULT nextval('public.watched_logs_id_seq'::regclass);
+
+
+--
+-- Name: addresses addresses_address_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.addresses
+    ADD CONSTRAINT addresses_address_key UNIQUE (address);
+
+
+--
+-- Name: addresses addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.addresses
+    ADD CONSTRAINT addresses_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -715,6 +841,14 @@ ALTER TABLE ONLY public.eth_nodes
     ADD CONSTRAINT eth_node_uc UNIQUE (genesis_block, network_id, eth_node_id);
 
 
+--
+-- Name: full_sync_logs full_sync_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.full_sync_logs
+    ADD CONSTRAINT full_sync_logs_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: full_sync_receipts full_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -739,6 +873,22 @@ ALTER TABLE ONLY public.goose_db_version
     ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: header_sync_logs header_sync_logs_header_id_tx_index_log_index_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_logs
+    ADD CONSTRAINT header_sync_logs_header_id_tx_index_log_index_key UNIQUE (header_id, tx_index, log_index);
+
+
+--
+-- Name: header_sync_logs header_sync_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_logs
+    ADD CONSTRAINT header_sync_logs_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -771,6 +921,14 @@ ALTER TABLE ONLY public.header_sync_transactions
     ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: headers headers_block_number_hash_eth_node_fingerprint_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.headers
+    ADD CONSTRAINT headers_block_number_hash_eth_node_fingerprint_key UNIQUE (block_number, hash, eth_node_fingerprint);
+
+
 --
 -- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -779,14 +937,6 @@ ALTER TABLE ONLY public.headers
     ADD CONSTRAINT headers_pkey PRIMARY KEY (id);
 
 
---
--- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.logs
-    ADD CONSTRAINT logs_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: log_filters name_uc; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -803,6 +953,14 @@ ALTER TABLE ONLY public.eth_nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: queued_storage queued_storage_block_height_block_hash_contract_storage_key_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.queued_storage
+    ADD CONSTRAINT queued_storage_block_height_block_hash_contract_storage_key_key UNIQUE (block_height, block_hash, contract, storage_key, storage_value);
+
+
 --
 -- Name: queued_storage queued_storage_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -843,6 +1001,14 @@ ALTER TABLE ONLY public.watched_contracts
     ADD CONSTRAINT watched_contracts_pkey PRIMARY KEY (contract_id);
 
 
+--
+-- Name: watched_logs watched_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.watched_logs
+    ADD CONSTRAINT watched_logs_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: block_id_index; Type: INDEX; Schema: public; Owner: -
 --
@@ -901,6 +1067,14 @@ ALTER TABLE ONLY public.checked_headers
     ADD CONSTRAINT checked_headers_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
 
 
+--
+-- Name: full_sync_receipts full_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.full_sync_receipts
+    ADD CONSTRAINT full_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
+
+
 --
 -- Name: full_sync_transactions full_sync_transactions_block_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -909,6 +1083,30 @@ ALTER TABLE ONLY public.full_sync_transactions
     ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
 
 
+--
+-- Name: header_sync_logs header_sync_logs_address_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_logs
+    ADD CONSTRAINT header_sync_logs_address_fkey FOREIGN KEY (address) REFERENCES public.addresses(id) ON DELETE CASCADE;
+
+
+--
+-- Name: header_sync_logs header_sync_logs_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_logs
+    ADD CONSTRAINT header_sync_logs_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
+--
+-- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
+
+
 --
 -- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -950,10 +1148,10 @@ ALTER TABLE ONLY public.blocks
 
 
 --
--- Name: logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
+-- Name: full_sync_logs receipts_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
 
-ALTER TABLE ONLY public.logs
+ALTER TABLE ONLY public.full_sync_logs
     ADD CONSTRAINT receipts_fk FOREIGN KEY (receipt_id) REFERENCES public.full_sync_receipts(id) ON DELETE CASCADE;
 
 

documentation/contributing.md

@@ -1,11 +1,50 @@
 # Contribution guidelines
 
-Contributions are welcome! In addition to core contributions, developers are encouraged to build their own custom transformers which
+Contributions are welcome! Please open an Issues or Pull Request for any changes.
-can be run together with other custom transformers using the [composeAndExeucte](../../staging/documentation/composeAndExecute.md) command.
+
+In addition to core contributions, developers are encouraged to build their own custom transformers which
+can be run together with other custom transformers using the [composeAndExecute](../../staging/documentation/custom-transformers.md) command.
+
+## Pull Requests
+- `go fmt` is run as part of `make test` and `make integrationtest`, please make sure to check in the format changes.
+- Ensure that new code is well tested, including integration testing if applicable.
+- Make sure the build is passing.
+- Update the README or any [documentation files](./) as necessary. If editing the Readme, please
+conform to the
+[standard-readme specification](https://github.com/RichardLitt/standard-readme).
+- Once a Pull Request has received two approvals it can be merged in by a core developer.
 
 ## Creating a new migration file
 1. `make new_migration NAME=add_columnA_to_table1`
     - This will create a new timestamped migration file in `db/migrations`
 1. Write the migration code in the created file, under the respective `goose` pragma
     - Goose automatically runs each migration in a transaction; don't add `BEGIN` and `COMMIT` statements.
-1. Core migrations should be committed in their `goose fix`ed form.
+1. Core migrations should be committed in their `goose fix`ed form. To do this, run `make version_migrations` which
+converts timestamped migrations to migrations versioned by an incremented integer.
+
+## Diagrams
+- Diagrams were created with [draw.io](https://www.draw.io).
+- To update a diagram:
+  1. Go to [draw.io](https://www.draw.io).
+  1. Click on *File > Open from* and choose the location of the diagram you want to update.
+  1. Once open in draw.io, you may update it.
+  1. Export the diagram to this repository's directory and commit it.
+
+
+## Generating the Changelog
+We use [github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) to generate release Changelogs. To be consistent with previous Changelogs, the following flags should be passed to the command:
+
+```
+--user vulcanize
+--project vulcanizedb
+--token {YOUR_GITHUB_TOKEN}
+--no-issues
+--usernames-as-github-logins
+--since-tag {PREVIOUS_RELEASE_TAG}
+```
+
+For more information on why your github token is needed, and how to generate it see [https://github
+.com/github-changelog-generator/github-changelog-generator#github-token](https://github.com/github-changelog-generator/github-changelog-generator#github-token).
+
+## Code of Conduct
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).

documentation/custom-transformers.md

@@ -1,44 +1,80 @@
-# composeAndExecute
+# Custom Transformers
-The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers.
+When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
-This is accomplished by generating a Go pluggin which allows the `vulcanizedb` binary to link to external transformers, so
+purpose can be leveraged.
-long as they abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
 
-Additionally, there are separate `compose` and `execute` commands to allow pre-building and linking to a pre-built .so file.
+Individual custom transformers can be composed together from any number of external repositories and executed as a
+single process using the `compose` and `execute` commands or the `composeAndExecute` command. This is accomplished by
+generating a Go plugin which allows the `vulcanizedb` binary to link to the external transformers, so long as they
+abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
 
-**NOTE:**
+## Writing custom transformers
-1. It is necessary that the .so file was built with the same exact dependencies that are present in the execution environment,
+For help with writing different types of custom transformers please see below:
-i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
-1. The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so in a different
-environment than the one it was composed in then the migrations for that plugin will first need to be manually ran against that environment's Postgres database.
 
-These commands require Go 1.11+ and use [Go plugins](https://golang.org/pkg/plugin/) which only work on Unix-based systems.
+Storage Transformers: transform data derived from contract storage tries
-There is also an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use vendored dependencies which
+   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
-imposes certain limitations on how the plugins are built.
+   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
 
-## Commands
+Event Transformers: transform data derived from Ethereum log events
-The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's `$GOPATH`,
+   * [Guide](../../staging/libraries/shared/factories/event/README.md)
-and that all of the transformer repositories for building the plugin are present at their `$GOPATH` directories.
+   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
+   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
+   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
 
-The `execute` command does not require the plugin transformer dependencies be located in their
+Contract Transformers: transform data derived from Ethereum log events and use it to poll public contract methods
-`$GOPATH` directories, instead it expects a prebuilt .so file (of the name specified in the config file)
+   * [Example 1](https://github.com/vulcanize/account_transformers)
-to be in `$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin
+   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
-db migrations to have already been ran against the database.
 
-compose:
+## Preparing custom transformers to work as part of a plugin
+To plug in an external transformer we need to:
 
-`./vulcanizedb compose --config=./environments/config_name.toml`
+1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
+or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
+or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
+2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
+[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
+or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
+3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
+    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
+    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
+    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
+    as they are [here](../../staging/db/migrations)
 
-execute:
+To update a plugin repository with changes to the core vulcanizedb repository, use your dependency manager to install the desired version of vDB.
 
-`./vulcanizedb execute --config=./environments/config_name.toml`
+## Building and Running Custom Transformers
+### Commands
+* The `compose`, `execute`, `composeAndExecute` commands require Go 1.11+ and use [Go plugins](https://golang
+.org/pkg/plugin/) which only work on Unix-based systems.
 
-composeAndExecute:
+* There is an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use of vendored
+dependencies which imposes certain limitations on how the plugins are built.
 
-`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
+* Separate `compose` and `execute` commands allow pre-building and linking to the pre-built .so file. So, if
+these are run independently, instead of using `composeAndExecute`, a couple of things need to be considered:
+    * It is necessary that the .so file was built with the same exact dependencies that are present in the execution
+    environment, i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
+    * The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so
+    in a different environment than the one it was composed in, then the database structure will need to be loaded 
+    into the environment's Postgres database. This can either be done by manually loading the plugin's schema into 
+    Postgres, or by manually running the plugin's migrations.
+     
+* The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's 
+`$GOPATH`, and that the plugin dependencies are present at their `$GOPATH` directories.
 
-## Flags
+* The `execute` command does not require the plugin transformer dependencies be located in their `$GOPATH` directories,
+instead it expects a .so file (of the name specified in the config file) to be in
+`$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin db migrations to
+ have already been ran against the database.
 
-The `compose` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
+ * Usage:
+     * compose: `./vulcanizedb compose --config=environments/config_name.toml`
+
+     * execute: `./vulcanizedb execute --config=environments/config_name.toml`
+
+     * composeAndExecute: `./vulcanizedb composeAndExecute --config=environments/config_name.toml`
+
+### Flags
+The `execute` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
 
 - `--recheck-headers`/`-r` - specifies whether to re-check headers for events after the header has already been queried for watched logs.
 Can be useful for redundancy if you suspect that your node is not always returning all desired logs on every query.
@@ -50,7 +86,7 @@ Defaults to `false`.
 Argument is expected to be a duration (integer measured in nanoseconds): e.g. `-q=10m30s` (for 10 minute, 30 second intervals).
 Defaults to `5m` (5 minutes).
 
-## Configuration
+### Configuration
 A .toml config file is specified when executing the commands.
 The config provides information for composing a set of transformers from external repositories:
 

documentation/data-syncing.md

@@ -0,0 +1,81 @@
+# Syncing commands
+These commands are used to sync raw Ethereum data into Postgres, with varying levels of data granularity.
+
+## headerSync
+Syncs block headers from a running Ethereum node into the VulcanizeDB table `headers`.
+- Queries the Ethereum node using RPC calls.
+- Validates headers from the last 15 blocks to ensure that data is up to date.
+- Useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart
+contract storage values or event logs).
+- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
+different from what we have already stored in the database, the header record will be updated.
+
+#### Usage
+- Run: `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
+- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    ipcPath  = <path to a running Ethereum node>
+```
+- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
+
+## fullSync
+Syncs blocks, transactions, receipts and logs from a running Ethereum node into VulcanizeDB tables named
+`blocks`, `uncles`, `full_sync_transactions`, `full_sync_receipts` and `logs`. 
+- Queries the Ethereum node using RPC calls.
+- Validates headers from the last 15 blocks to ensure that data is up to date.
+- Useful when you want to maintain a broad cache of what's happening on the blockchain.
+- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
+different from what we have already stored in the database, the header record will be updated.
+
+#### Usage
+- Run `./vulcanizedb fullSync --config <config.toml> --starting-block-number <block-number>`
+- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    ipcPath  = <path to a running Ethereum node>
+```
+- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
+
+*Please note, that if you are fast syncing your Ethereum node, wait for the initial sync to finish.*
+
+## coldImport
+Syncs VulcanizeDB from Geth's underlying LevelDB datastore and persists Ethereum blocks, 
+transactions, receipts and logs into VulcanizeDB tables named `blocks`, `uncles`, 
+`full_sync_transactions`, `full_sync_receipts` and `logs` respectively.
+
+#### Usage
+1. Ensure the Ethereum node you're point at is not running, and that it has synced to the desired block height.
+1. Run `./vulcanizedb coldImport --config <config.toml>`
+1. Optional flags:
+    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
+    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
+    - `--all`/`-a`: sync all missing blocks
+
+The config file can be formatted as follows, and must contain the LevelDB path.
+
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    leveldbpath = "/Users/user/Library/Ethereum/geth/chaindata"
+```

documentation/generic-transformer.md

@@ -1,4 +1,4 @@
-# contractWatcher
+# Generic Transformer
 The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available.
 It also provides some state variable coverage by automating polling of public methods, with some restrictions:
 1. The method must have 2 or less arguments

documentation/postgraphile.md

@@ -0,0 +1,34 @@
+# Postgraphile
+
+You can expose VulcanizeDB data via [Postgraphile](https://github.com/graphile/postgraphile).
+Check out [their documentation](https://www.graphile.org/postgraphile/) for the most up-to-date instructions on installing, running, and customizing Postgraphile.
+
+## Simple Setup
+
+As of April 30, 2019, you can run Postgraphile pointed at the default `vulcanize_public` database with the following commands:
+
+```
+npm install -g postgraphile
+postgraphile --connection postgres://localhost/vulcanize_public --schema=public,custom --disable-default-mutations --no-ignore-rbac
+```
+
+Arguments:
+- `--connection` specifies the database. The above command connects to the default `vulcanize_public` database 
+defined in [the example config](../environments/public.toml.example).
+- `--schema` defines what schema(s) to expose. The above exposes the `public` schema (for core VulcanizeDB data) as well as a `custom` schema (where `custom` is the name of a schema defined in executed transformers).
+- `--disable-default-mutations` prevents Postgraphile from exposing create, update, and delete operations on your data, which are otherwise enabled by default.
+- `--no-ignore-rbac` ensures that Postgraphile will only expose the tables, columns, fields, and query functions that the user has explicit access to.
+
+## Customizing Postgraphile
+
+By default, Postgraphile will expose queries for all tables defined in your chosen database/schema(s), including [filtering](https://www.graphile.org/postgraphile/filtering/) and [auto-discovered relations](https://www.graphile.org/postgraphile/relations/).
+
+If you'd like to expose more customized windows into your data, there are some techniques you can apply when writing migrations:
+
+- [Computed columns](https://www.graphile.org/postgraphile/computed-columns/) enable you to derive additional fields from types defined in your database.
+For example, you could write a function to expose a block header's state root over Postgraphile with a computed column - without modifying the `public.headers` table.
+- [Custom queries](https://www.graphile.org/postgraphile/custom-queries/) enable you to provide on-demand access to more complex data (e.g. the product of joining and filtering several tables' data based on a passed argument).
+For example, you could write a custom query to get the block timestamp for every transaction originating from a given address.
+- [Subscriptions](https://www.graphile.org/postgraphile/subscriptions/) enable you to publish data as it is coming into your database.
+
+The above list is not exhaustive - please see the Postgraphile documentation for a more comprehensive and up-to-date description of available features.

documentation/sync.md

@@ -1,26 +0,0 @@
-# Syncing commands
-These commands are used to sync raw Ethereum data into Postgres.
-
-## headerSync
-Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
-This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values or event logs).
-1. Start Ethereum node
-1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
-    
-## sync
-Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.
-This command is useful when you want to maintain a broad cache of what's happening on the blockchain.
-1. Start Ethereum node (**if fast syncing your Ethereum node, wait for initial sync to finish**)
-1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb sync --config <config.toml> --starting-block-number <block-number>`
-
-## coldImport
-Sync VulcanizeDB from the LevelDB underlying a Geth node.
-1. Assure node is not running, and that it has synced to the desired block height.
-1. Start vulcanize_db
-   - `./vulcanizedb coldImport --config <config.toml>`
-1. Optional flags:
-    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
-    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
-    - `--all`/`-a`: sync all missing blocks

documentation/transformers.md

@@ -1,40 +0,0 @@
-# Custom transformers
-When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
-purpose can be leveraged.
-
-Individual transformers can be composed together from any number of external repositories and executed as a single process using
-the `compose` and `execute` commands or the `composeAndExecute` command.
-
-## Writing custom transformers
-For help with writing different types of custom transformers for the `composeAndExecute` set of commands, please see the below:
-
-Storage Transformers
-   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
-   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
-
-Event Transformers
-   * [Guide](../../staging/libraries/shared/factories/event/README.md)
-   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
-   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
-
-Contract Transformers
-   * [Example 1](https://github.com/vulcanize/account_transformers)
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
-
-## Preparing custom transformers to work as part of a plugin
-To plug in an external transformer we need to:
-
-1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
-or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
-or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
-2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
-[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
-or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
-3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
-    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
-    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
-    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
-    as they are [here](../../staging/db/migrations)
-
-To update a plugin repository with changes to the core vulcanizedb repository, run `dep ensure` to update its dependencies.

go.mod

@@ -0,0 +1,57 @@
+module github.com/vulcanize/vulcanizedb
+
+go 1.12
+
+require (
+	github.com/allegro/bigcache v1.2.1 // indirect
+	github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 // indirect
+	github.com/dave/jennifer v1.3.0
+	github.com/deckarep/golang-set v1.7.1 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/elastic/gosigar v0.10.4 // indirect
+	github.com/ethereum/go-ethereum v1.9.5
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.3
+	github.com/howeyc/fsnotify v0.9.0 // indirect
+	github.com/hpcloud/tail v1.0.0
+	github.com/huin/goupnp v1.0.0 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
+	github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a
+	github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 // indirect
+	github.com/lib/pq v1.0.0
+	github.com/mattn/go-colorable v0.1.2 // indirect
+	github.com/mattn/go-isatty v0.0.9 // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/olekukonko/tablewriter v0.0.1 // indirect
+	github.com/onsi/ginkgo v1.7.0
+	github.com/onsi/gomega v1.4.3
+	github.com/pborman/uuid v1.2.0 // indirect
+	github.com/pressly/goose v2.6.0+incompatible
+	github.com/prometheus/tsdb v0.10.0 // indirect
+	github.com/rjeczalik/notify v0.9.2 // indirect
+	github.com/rs/cors v1.7.0 // indirect
+	github.com/sirupsen/logrus v1.2.0
+	github.com/spf13/cobra v0.0.3
+	github.com/spf13/viper v1.3.2
+	github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect
+	github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 // indirect
+	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/tyler-smith/go-bip39 v1.0.2 // indirect
+	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
+	golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 // indirect
+	golang.org/x/net v0.0.0-20190603091049-60506f45cf65
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+)
+
+replace github.com/ethereum/go-ethereum => github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101
+
+replace gopkg.in/urfave/cli.v1 => gopkg.in/urfave/cli.v1 v1.20.0

go.sum

@@ -0,0 +1,318 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc=
+github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A=
+github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU=
+github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
+github.com/aristanetworks/goarista v0.0.0-20190121184617-8f049bdb8feb/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
+github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
+github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
+github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw=
+github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
+github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
+github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
+github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
+github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
+github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
+github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
+github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
+github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo=
+github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
+github.com/ethereum/go-ethereum v1.9.0/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
+github.com/ethereum/go-ethereum v1.9.1 h1:MrdTRvKIa3apdx6NW1azzSgl8BQB1eTBVSUmFhuztaU=
+github.com/ethereum/go-ethereum v1.9.1/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
+github.com/ethereum/go-ethereum v1.9.5 h1:4oxsF+/3N/sTgda9XTVG4r+wMVLsveziSMcK83hPbsk=
+github.com/ethereum/go-ethereum v1.9.5/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
+github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
+github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 h1:9WiNlI9Cds5S5YITwRpRs8edNaq0nxTEymhDW20A1QE=
+github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
+github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
+github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
+github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
+github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb v1.7.7 h1:UvNzAPfBrKMENVbQ4mr4ccA9sW+W1Ihl0Yh1s0BiVAg=
+github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
+github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
+github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw=
+github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68=
+github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo=
+github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
+github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
+github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.9.1/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
+github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic=
+github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
+github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
+github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
+github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
+github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
+github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
+github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
+github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
+github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
+github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
+github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM=
+github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 h1:GnOzE5fEFN3b2zDhJJABEofdb51uMRNb8eqIVtdducs=
+github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
+github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8=
+github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101 h1:fsHhBzscAwi4u7/F033SFJwTIz+46D8uDWMu2/ZdvzA=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug=
+github.com/vulcanize/go-ethereum v0.0.0-20190731183759-8e20673bd101/go.mod h1:9i0pGnKDUFFr8yC/n8xyrNBVfhYlpwE8J3Ge6ThKvug=
+github.com/vulcanize/vulcanizedb v0.0.5/go.mod h1:utXkheCL9VjTfmuivuvRiAAyHh54GSN9XRQNEbFCA8k=
+github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
+github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794 h1:4Yo9XtTfxfBCecLiBW8TYsFIdN7TkDhjGLWetFo4JSo=
+golang.org/x/crypto v0.0.0-20190926114937-fa1a29108794/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

integration_test/block_rewards_test.go

@@ -22,10 +22,10 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	vRpc "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -39,7 +39,7 @@ var _ = Describe("Rewards calculations", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-		blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+		blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.Reward).To(Equal("5313550000000000000"))
@@ -53,7 +53,7 @@ var _ = Describe("Rewards calculations", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := vRpc.NewRpcTransactionConverter(ethClient)
-		blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+		blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 		block, err := blockChain.GetBlockByNumber(1071819)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(block.UnclesReward).To(Equal("6875000000000000000"))

integration_test/contract_test.go

@@ -26,11 +26,11 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/testing"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/testing"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
@@ -38,7 +38,7 @@ var _ = Describe("Reading contracts", func() {
 
 	Describe("Getting a contract attribute", func() {
 		It("retrieves the event log for a specific block and contract", func() {
-			expectedLogZero := core.Log{
+			expectedLogZero := core.FullSyncLog{
 				BlockNumber: 4703824,
 				TxHash:      "0xf896bfd1eb539d881a1a31102b78de9f25cd591bf1fe1924b86148c0b205fd5d",
 				Address:     "0xd26114cd6ee289accf82350c8d8487fedb8a0c07",
@@ -56,10 +56,10 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 			contract := testing.SampleContract()
 
-			logs, err := blockChain.GetLogs(contract, big.NewInt(4703824), nil)
+			logs, err := blockChain.GetFullSyncLogs(contract, big.NewInt(4703824), nil)
 
 			Expect(err).To(BeNil())
 			Expect(len(logs)).To(Equal(3))
@@ -74,9 +74,9 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
-			logs, err := blockChain.GetLogs(core.Contract{Hash: "0x123"}, big.NewInt(4703824), nil)
+			logs, err := blockChain.GetFullSyncLogs(core.Contract{Hash: "0x123"}, big.NewInt(4703824), nil)
 
 			Expect(err).To(BeNil())
 			Expect(len(logs)).To(Equal(0))
@@ -92,7 +92,7 @@ var _ = Describe("Reading contracts", func() {
 			blockChainClient := client.NewEthClient(ethClient)
 			node := node.MakeNode(rpcClient)
 			transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-			blockChain := geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+			blockChain := eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 
 			contract := testing.SampleContract()
 			var balance = new(big.Int)

integration_test/contract_watcher_full_transformer_test.go

@@ -1,8 +1,23 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package integration
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"math/rand"
 	"strings"
 	"time"
@@ -11,6 +26,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

integration_test/contract_watcher_header_sync_transformer_test.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package integration
 
 import (
@@ -24,8 +40,8 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 	var blockChain core.BlockChain
 	var headerRepository repositories.HeaderRepository
 	var headerID int64
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)   // 0x314159265dd8dbb310642f98f50c066173c1259b
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e
 
 	BeforeEach(func() {
 		db, blockChain = test_helpers.SetupDBandBC()
@@ -62,11 +78,10 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 			Expect(c.Address).To(Equal(tusdAddr))
 		})
 
-		It("Fails to initialize if first and block cannot be fetched from vDB headers table", func() {
+		It("initializes when no headers available in db", func() {
 			t := transformer.NewTransformer(test_helpers.TusdConfig, blockChain, db)
 			err = t.Init()
-			Expect(err).To(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
 
 		It("Does nothing if nothing if no addresses are configured", func() {
@@ -362,6 +377,42 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 			Expect(transferLog.Value).To(Equal("2800000000000000000000"))
 		})
 
+		It("Marks header checked for a contract that has no logs at that header", func() {
+			t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t.Start).To(Equal(int64(6885702)))
+
+			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			Expect(err).ToNot(HaveOccurred())
+			transferLog := test_helpers.HeaderSyncTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID))
+
+			type checkedHeader struct {
+				ID       int64 `db:"id"`
+				HeaderID int64 `db:"header_id"`
+				NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"`
+				Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"`
+			}
+
+			transferCheckedHeader := new(checkedHeader)
+			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(transferCheckedHeader.Transfer).To(Equal(int64(1)))
+			Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1)))
+
+			newOwnerCheckedHeader := new(checkedHeader)
+			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1)))
+			Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1)))
+		})
+
 		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
 			var testConf config.ContractConfig
 			testConf = test_helpers.ENSandTusdConfig

integration_test/geth_blockchain_test.go

@@ -24,17 +24,17 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	rpc2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/pkg/geth"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
-	rpc2 "github.com/vulcanize/vulcanizedb/pkg/geth/converters/rpc"
-	"github.com/vulcanize/vulcanizedb/pkg/geth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/history"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
 
 var _ = Describe("Reading from the Geth blockchain", func() {
-	var blockChain *geth.BlockChain
+	var blockChain *eth.BlockChain
 
 	BeforeEach(func() {
 		rawRpcClient, err := rpc.Dial(test_config.InfuraClient.IPCPath)
@@ -44,7 +44,7 @@ var _ = Describe("Reading from the Geth blockchain", func() {
 		blockChainClient := client.NewEthClient(ethClient)
 		node := node.MakeNode(rpcClient)
 		transactionConverter := rpc2.NewRpcTransactionConverter(ethClient)
-		blockChain = geth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
+		blockChain = eth.NewBlockChain(blockChainClient, rpcClient, node, transactionConverter)
 	})
 
 	It("reads two blocks", func(done Done) {

integration_test/integration_test_suite_test.go

@@ -17,6 +17,8 @@
 package integration_test
 
 import (
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -27,3 +29,7 @@ func TestIntegrationTest(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "IntegrationTest Suite")
 }
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

libraries/shared/chunker/log_chunker.go

@@ -17,17 +17,15 @@
 package chunker
 
 import (
-	"strings"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-
+	"github.com/vulcanize/vulcanizedb/pkg/core"
-	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"strings"
 )
 
 type Chunker interface {
-	AddConfigs(transformerConfigs []shared_t.EventTransformerConfig)
+	AddConfig(transformerConfig transformer.EventTransformerConfig)
-	ChunkLogs(logs []types.Log) map[string][]types.Log
+	ChunkLogs(logs []core.HeaderSyncLog) map[string][]core.HeaderSyncLog
 }
 
 type LogChunker struct {
@@ -44,27 +42,25 @@ func NewLogChunker() *LogChunker {
 	}
 }
 
-// Configures the chunker by adding more addreses and topics to consider.
+// Configures the chunker by adding one config with more addresses and topics to consider.
-func (chunker *LogChunker) AddConfigs(transformerConfigs []shared_t.EventTransformerConfig) {
+func (chunker *LogChunker) AddConfig(transformerConfig transformer.EventTransformerConfig) {
-	for _, config := range transformerConfigs {
+	for _, address := range transformerConfig.ContractAddresses {
-		for _, address := range config.ContractAddresses {
+		var lowerCaseAddress = strings.ToLower(address)
-			var lowerCaseAddress = strings.ToLower(address)
+		chunker.AddressToNames[lowerCaseAddress] = append(chunker.AddressToNames[lowerCaseAddress], transformerConfig.TransformerName)
-			chunker.AddressToNames[lowerCaseAddress] = append(chunker.AddressToNames[lowerCaseAddress], config.TransformerName)
+		chunker.NameToTopic0[transformerConfig.TransformerName] = common.HexToHash(transformerConfig.Topic)
-			chunker.NameToTopic0[config.TransformerName] = common.HexToHash(config.Topic)
-		}
 	}
 }
 
-// Goes through an array of logs, associating relevant logs (matching addresses and topic) with transformers
+// Goes through a slice of logs, associating relevant logs (matching addresses and topic) with transformers
-func (chunker *LogChunker) ChunkLogs(logs []types.Log) map[string][]types.Log {
+func (chunker *LogChunker) ChunkLogs(logs []core.HeaderSyncLog) map[string][]core.HeaderSyncLog {
-	chunks := map[string][]types.Log{}
+	chunks := map[string][]core.HeaderSyncLog{}
 	for _, log := range logs {
 		// Topic0 is not unique to each transformer, also need to consider the contract address
-		relevantTransformers := chunker.AddressToNames[strings.ToLower(log.Address.String())]
+		relevantTransformers := chunker.AddressToNames[strings.ToLower(log.Log.Address.Hex())]
 
-		for _, transformer := range relevantTransformers {
+		for _, t := range relevantTransformers {
-			if chunker.NameToTopic0[transformer] == log.Topics[0] {
+			if chunker.NameToTopic0[t] == log.Log.Topics[0] {
-				chunks[transformer] = append(chunks[transformer], log)
+				chunks[t] = append(chunks[t], log)
 			}
 		}
 	}

libraries/shared/chunker/log_chunker_test.go

@@ -21,38 +21,39 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	chunk "github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
-	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 var _ = Describe("Log chunker", func() {
 	var (
-		configs []shared_t.EventTransformerConfig
 		chunker *chunk.LogChunker
 	)
 
 	BeforeEach(func() {
-		configA := shared_t.EventTransformerConfig{
+		chunker = chunk.NewLogChunker()
+
+		configA := transformer.EventTransformerConfig{
 			TransformerName:   "TransformerA",
 			ContractAddresses: []string{"0x00000000000000000000000000000000000000A1", "0x00000000000000000000000000000000000000A2"},
 			Topic:             "0xA",
 		}
-		configB := shared_t.EventTransformerConfig{
+		chunker.AddConfig(configA)
+
+		configB := transformer.EventTransformerConfig{
 			TransformerName:   "TransformerB",
 			ContractAddresses: []string{"0x00000000000000000000000000000000000000B1"},
 			Topic:             "0xB",
 		}
+		chunker.AddConfig(configB)
 
-		configC := shared_t.EventTransformerConfig{
+		configC := transformer.EventTransformerConfig{
 			TransformerName:   "TransformerC",
 			ContractAddresses: []string{"0x00000000000000000000000000000000000000A2"},
 			Topic:             "0xC",
 		}
-
+		chunker.AddConfig(configC)
-		configs = []shared_t.EventTransformerConfig{configA, configB, configC}
-		chunker = chunk.NewLogChunker()
-		chunker.AddConfigs(configs)
 	})
 
 	Describe("initialisation", func() {
@@ -71,26 +72,26 @@ var _ = Describe("Log chunker", func() {
 		})
 	})
 
-	Describe("AddConfigs", func() {
+	Describe("AddConfig", func() {
 		It("can add more configs later", func() {
-			configD := shared_t.EventTransformerConfig{
+			configD := transformer.EventTransformerConfig{
 				TransformerName:   "TransformerD",
 				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
 				Topic:             "0xD",
 			}
-			chunker.AddConfigs([]shared_t.EventTransformerConfig{configD})
+			chunker.AddConfig(configD)
 
 			Expect(chunker.AddressToNames).To(ContainElement([]string{"TransformerD"}))
 			Expect(chunker.NameToTopic0).To(ContainElement(common.HexToHash("0xD")))
 		})
 
 		It("lower cases address", func() {
-			configD := shared_t.EventTransformerConfig{
+			configD := transformer.EventTransformerConfig{
 				TransformerName:   "TransformerD",
 				ContractAddresses: []string{"0x000000000000000000000000000000000000000D"},
 				Topic:             "0xD",
 			}
-			chunker.AddConfigs([]shared_t.EventTransformerConfig{configD})
+			chunker.AddConfig(configD)
 
 			Expect(chunker.AddressToNames["0x000000000000000000000000000000000000000d"]).To(Equal([]string{"TransformerD"}))
 		})
@@ -98,7 +99,7 @@ var _ = Describe("Log chunker", func() {
 
 	Describe("ChunkLogs", func() {
 		It("only associates logs with relevant topic0 and address to transformers", func() {
-			logs := []types.Log{log1, log2, log3, log4, log5}
+			logs := []core.HeaderSyncLog{log1, log2, log3, log4, log5}
 			chunks := chunker.ChunkLogs(logs)
 
 			Expect(chunks["TransformerA"]).To(And(ContainElement(log1), ContainElement(log4)))
@@ -110,43 +111,53 @@ var _ = Describe("Log chunker", func() {
 
 var (
 	// Match TransformerA
-	log1 = types.Log{
+	log1 = core.HeaderSyncLog{
-		Address: common.HexToAddress("0xA1"),
+		Log: types.Log{
-		Topics: []common.Hash{
+			Address: common.HexToAddress("0xA1"),
-			common.HexToHash("0xA"),
+			Topics: []common.Hash{
-			common.HexToHash("0xLogTopic1"),
+				common.HexToHash("0xA"),
+				common.HexToHash("0xLogTopic1"),
+			},
 		},
 	}
 	// Match TransformerA address, but not topic0
-	log2 = types.Log{
+	log2 = core.HeaderSyncLog{
-		Address: common.HexToAddress("0xA1"),
+		Log: types.Log{
-		Topics: []common.Hash{
+			Address: common.HexToAddress("0xA1"),
-			common.HexToHash("0xB"),
+			Topics: []common.Hash{
-			common.HexToHash("0xLogTopic2"),
+				common.HexToHash("0xB"),
+				common.HexToHash("0xLogTopic2"),
+			},
 		},
 	}
 	// Match TransformerA topic, but TransformerB address
-	log3 = types.Log{
+	log3 = core.HeaderSyncLog{
-		Address: common.HexToAddress("0xB1"),
+		Log: types.Log{
-		Topics: []common.Hash{
+			Address: common.HexToAddress("0xB1"),
-			common.HexToHash("0xA"),
+			Topics: []common.Hash{
-			common.HexToHash("0xLogTopic3"),
+				common.HexToHash("0xA"),
+				common.HexToHash("0xLogTopic3"),
+			},
 		},
 	}
 	// Match TransformerA, with the other address
-	log4 = types.Log{
+	log4 = core.HeaderSyncLog{
-		Address: common.HexToAddress("0xA2"),
+		Log: types.Log{
-		Topics: []common.Hash{
+			Address: common.HexToAddress("0xA2"),
-			common.HexToHash("0xA"),
+			Topics: []common.Hash{
-			common.HexToHash("0xLogTopic4"),
+				common.HexToHash("0xA"),
+				common.HexToHash("0xLogTopic4"),
+			},
 		},
 	}
 	// Match TransformerC, which shares address with TransformerA
-	log5 = types.Log{
+	log5 = core.HeaderSyncLog{
-		Address: common.HexToAddress("0xA2"),
+		Log: types.Log{
-		Topics: []common.Hash{
+			Address: common.HexToAddress("0xA2"),
-			common.HexToHash("0xC"),
+			Topics: []common.Hash{
-			common.HexToHash("0xLogTopic5"),
+				common.HexToHash("0xC"),
+				common.HexToHash("0xLogTopic5"),
+			},
 		},
 	}
 )

libraries/shared/constants/checked_headers.go

@@ -20,6 +20,6 @@ type TransformerExecution bool
 
 const (
 	HeaderRecheck    TransformerExecution = true
-	HeaderMissing    TransformerExecution = false
+	HeaderUnchecked  TransformerExecution = false
-	RecheckHeaderCap                      = "4"
+	RecheckHeaderCap                      = int64(5)
 )

libraries/shared/factories/event/converter.go

@@ -16,9 +16,13 @@
 
 package event
 
-import "github.com/ethereum/go-ethereum/core/types"
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
 
+// Converter transforms log data into general InsertionModels the Repository can persist__
 type Converter interface {
-	ToEntities(contractAbi string, ethLog []types.Log) ([]interface{}, error)
+	ToModels(contractAbi string, ethLog []core.HeaderSyncLog) ([]InsertionModel, error)
-	ToModels([]interface{}) ([]interface{}, error)
+	SetDB(db *postgres.DB)
 }

libraries/shared/factories/event/repository.go

@@ -17,14 +17,162 @@
 package event
 
 import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"database/sql/driver"
+	"fmt"
+	"github.com/vulcanize/vulcanizedb/utils"
+	"strings"
+
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+const SetLogTransformedQuery = `UPDATE public.header_sync_logs SET transformed = true WHERE id = $1`
+
+// Repository persists transformed values to the DB
 type Repository interface {
-	Create(headerID int64, models []interface{}) error
+	Create(models []InsertionModel) error
-	MarkHeaderChecked(headerID int64) error
-	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
-	RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
 	SetDB(db *postgres.DB)
 }
+
+// LogFK is the name of log foreign key columns
+const LogFK ColumnName = "log_id"
+
+// AddressFK is the name of address foreign key columns
+const AddressFK ColumnName = "address_id"
+
+// HeaderFK is the name of header foreign key columns
+const HeaderFK ColumnName = "header_id"
+
+// SchemaName is the schema to work with
+type SchemaName string
+
+// TableName identifies the table for inserting the data
+type TableName string
+
+// ColumnName identifies columns on the given table
+type ColumnName string
+
+// ColumnValues maps a column to the value for insertion. This is restricted to []byte, bool, float64, int64, string, time.Time
+type ColumnValues map[ColumnName]interface{}
+
+// ErrUnsupportedValue is thrown when a model supplies a type of value the postgres driver cannot handle.
+var ErrUnsupportedValue = func(value interface{}) error {
+	return fmt.Errorf("unsupported type of value supplied in model: %v (%T)", value, value)
+}
+
+// InsertionModel is the generalised data structure a converter returns, and contains everything the repository needs to
+// persist the converted data.
+type InsertionModel struct {
+	SchemaName     SchemaName
+	TableName      TableName
+	OrderedColumns []ColumnName // Defines the fields to insert, and in which order the table expects them
+	ColumnValues   ColumnValues // Associated values for columns, restricted to []byte, bool, float64, int64, string, time.Time
+}
+
+// ModelToQuery stores memoised insertion queries to minimise computation
+var ModelToQuery = map[string]string{}
+
+// GetMemoizedQuery gets/creates a DB insertion query for the model
+func GetMemoizedQuery(model InsertionModel) string {
+	// The schema and table name uniquely determines the insertion query, use that for memoization
+	queryKey := string(model.SchemaName) + string(model.TableName)
+	query, queryMemoized := ModelToQuery[queryKey]
+	if !queryMemoized {
+		query = GenerateInsertionQuery(model)
+		ModelToQuery[queryKey] = query
+	}
+	return query
+}
+
+// GenerateInsertionQuery creates an SQL insertion query from an insertion model.
+// Should be called through GetMemoizedQuery, so the query is not generated on each call to Create.
+func GenerateInsertionQuery(model InsertionModel) string {
+	var valuePlaceholders []string
+	var updateOnConflict []string
+	for i := 0; i < len(model.OrderedColumns); i++ {
+		valuePlaceholder := fmt.Sprintf("$%d", 1+i)
+		valuePlaceholders = append(valuePlaceholders, valuePlaceholder)
+		updateOnConflict = append(updateOnConflict,
+			fmt.Sprintf("%s = %s", model.OrderedColumns[i], valuePlaceholder))
+	}
+
+	baseQuery := `INSERT INTO %v.%v (%v) VALUES(%v)
+		ON CONFLICT (header_id, log_id) DO UPDATE SET %v;`
+
+	return fmt.Sprintf(baseQuery,
+		model.SchemaName,
+		model.TableName,
+		joinOrderedColumns(model.OrderedColumns),
+		strings.Join(valuePlaceholders, ", "),
+		strings.Join(updateOnConflict, ", "))
+}
+
+/*
+Create generates an insertion query and persists to the DB, given a slice of InsertionModels.
+ColumnValues are restricted to []byte, bool, float64, int64, string, time.Time.
+
+testModel = shared.InsertionModel{
+	SchemaName:     "public"
+	TableName:      "testEvent",
+	OrderedColumns: []string{"header_id", "log_id", "variable1"},
+	ColumnValues: ColumnValues{
+		"header_id": 303
+		"log_id":   "808",
+		"variable1": "value1",
+	},
+}
+*/
+func Create(models []InsertionModel, db *postgres.DB) error {
+	if len(models) == 0 {
+		return fmt.Errorf("repository got empty model slice")
+	}
+
+	tx, dbErr := db.Beginx()
+	if dbErr != nil {
+		return dbErr
+	}
+
+	for _, model := range models {
+		// Maps can't be iterated over in a reliable manner, so we rely on OrderedColumns to define the order to insert
+		// tx.Exec is variadically typed in the args, so if we wrap in []interface{} we can apply them all automatically
+		var args []interface{}
+		for _, col := range model.OrderedColumns {
+			value := model.ColumnValues[col]
+			// Check whether or not PG can accept the type of value in the model
+			okPgValue := driver.IsValue(value)
+			if !okPgValue {
+				logrus.WithField("model", model).Errorf("PG cannot handle value of this type: %T", value)
+				return ErrUnsupportedValue(value)
+			}
+			args = append(args, value)
+		}
+
+		insertionQuery := GetMemoizedQuery(model)
+		_, execErr := tx.Exec(insertionQuery, args...) // couldn't pass varying types in bulk with args :: []string
+
+		if execErr != nil {
+			rollbackErr := tx.Rollback()
+			if rollbackErr != nil {
+				logrus.Error("failed to rollback ", rollbackErr)
+			}
+			return execErr
+		}
+
+		_, logErr := tx.Exec(SetLogTransformedQuery, model.ColumnValues[LogFK])
+
+		if logErr != nil {
+			utils.RollbackAndLogFailure(tx, logErr, "header_sync_logs.transformed")
+			return logErr
+		}
+	}
+
+	return tx.Commit()
+}
+
+func joinOrderedColumns(columns []ColumnName) string {
+	var stringColumns []string
+	for _, columnName := range columns {
+		stringColumns = append(stringColumns, string(columnName))
+	}
+	return strings.Join(stringColumns, ", ")
+}

libraries/shared/factories/event/repository_test.go

@@ -0,0 +1,204 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package event_test
+
+import (
+	"fmt"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+	"math/big"
+)
+
+var _ = Describe("Repository", func() {
+	var db *postgres.DB
+
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+	})
+
+	Describe("Create", func() {
+		const createTestEventTableQuery = `CREATE TABLE public.testEvent(
+		id        SERIAL PRIMARY KEY,
+		header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
+		log_id    BIGINT  NOT NULL REFERENCES header_sync_logs (id) ON DELETE CASCADE,
+		variable1 TEXT,
+		UNIQUE (header_id, log_id)
+		);`
+
+		var (
+			headerID, logID  int64
+			headerRepository repositories.HeaderRepository
+			testModel        event.InsertionModel
+		)
+
+		BeforeEach(func() {
+			_, tableErr := db.Exec(createTestEventTableQuery)
+			Expect(tableErr).NotTo(HaveOccurred())
+			headerRepository = repositories.NewHeaderRepository(db)
+			var insertHeaderErr error
+			headerID, insertHeaderErr = headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
+			Expect(insertHeaderErr).NotTo(HaveOccurred())
+			headerSyncLog := test_data.CreateTestLog(headerID, db)
+			logID = headerSyncLog.ID
+
+			testModel = event.InsertionModel{
+				SchemaName: "public",
+				TableName:  "testEvent",
+				OrderedColumns: []event.ColumnName{
+					event.HeaderFK, event.LogFK, "variable1",
+				},
+				ColumnValues: event.ColumnValues{
+					event.HeaderFK: headerID,
+					event.LogFK:    logID,
+					"variable1":    "value1",
+				},
+			}
+		})
+
+		AfterEach(func() {
+			db.MustExec(`DROP TABLE public.testEvent;`)
+		})
+
+		// Needs to run before the other tests, since those insert keys in map
+		It("memoizes queries", func() {
+			Expect(len(event.ModelToQuery)).To(Equal(0))
+			event.GetMemoizedQuery(testModel)
+			Expect(len(event.ModelToQuery)).To(Equal(1))
+			event.GetMemoizedQuery(testModel)
+			Expect(len(event.ModelToQuery)).To(Equal(1))
+		})
+
+		It("persists a model to postgres", func() {
+			createErr := event.Create([]event.InsertionModel{testModel}, db)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var res TestEvent
+			dbErr := db.Get(&res, `SELECT log_id, variable1 FROM public.testEvent;`)
+			Expect(dbErr).NotTo(HaveOccurred())
+
+			Expect(res.LogID).To(Equal(fmt.Sprint(testModel.ColumnValues[event.LogFK])))
+			Expect(res.Variable1).To(Equal(testModel.ColumnValues["variable1"]))
+		})
+
+		Describe("returns errors", func() {
+			It("for empty model slice", func() {
+				err := event.Create([]event.InsertionModel{}, db)
+				Expect(err).To(MatchError("repository got empty model slice"))
+			})
+
+			It("for failed SQL inserts", func() {
+				header := fakes.GetFakeHeader(1)
+				headerID, headerErr := headerRepository.CreateOrUpdateHeader(header)
+				Expect(headerErr).NotTo(HaveOccurred())
+
+				brokenModel := event.InsertionModel{
+					SchemaName: "public",
+					TableName:  "testEvent",
+					// Wrong name of last column compared to DB, will generate incorrect query
+					OrderedColumns: []event.ColumnName{
+						event.HeaderFK, event.LogFK, "variable2",
+					},
+					ColumnValues: event.ColumnValues{
+						event.HeaderFK: headerID,
+						event.LogFK:    logID,
+						"variable1":    "value1",
+					},
+				}
+
+				// Remove cached queries, or we won't generate a new (incorrect) one
+				delete(event.ModelToQuery, "publictestEvent")
+
+				createErr := event.Create([]event.InsertionModel{brokenModel}, db)
+				// Remove incorrect query, so other tests won't get it
+				delete(event.ModelToQuery, "publictestEvent")
+
+				Expect(createErr).To(HaveOccurred())
+			})
+
+			It("for unsupported types in ColumnValue", func() {
+				unsupportedValue := big.NewInt(5)
+				testModel = event.InsertionModel{
+					SchemaName: "public",
+					TableName:  "testEvent",
+					OrderedColumns: []event.ColumnName{
+						event.HeaderFK, event.LogFK, "variable1",
+					},
+					ColumnValues: event.ColumnValues{
+						event.HeaderFK: headerID,
+						event.LogFK:    logID,
+						"variable1":    unsupportedValue,
+					},
+				}
+
+				createErr := event.Create([]event.InsertionModel{testModel}, db)
+				Expect(createErr).To(MatchError(event.ErrUnsupportedValue(unsupportedValue)))
+			})
+		})
+
+		It("upserts queries with conflicting source", func() {
+			conflictingModel := event.InsertionModel{
+				SchemaName: "public",
+				TableName:  "testEvent",
+				OrderedColumns: []event.ColumnName{
+					event.HeaderFK, event.LogFK, "variable1",
+				},
+				ColumnValues: event.ColumnValues{
+					event.HeaderFK: headerID,
+					event.LogFK:    logID,
+					"variable1":    "conflictingValue",
+				},
+			}
+
+			createErr := event.Create([]event.InsertionModel{testModel, conflictingModel}, db)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var res TestEvent
+			dbErr := db.Get(&res, `SELECT log_id, variable1 FROM public.testEvent;`)
+			Expect(dbErr).NotTo(HaveOccurred())
+			Expect(res.Variable1).To(Equal(conflictingModel.ColumnValues["variable1"]))
+		})
+
+		It("generates correct queries", func() {
+			actualQuery := event.GenerateInsertionQuery(testModel)
+			expectedQuery := `INSERT INTO public.testEvent (header_id, log_id, variable1) VALUES($1, $2, $3)
+		ON CONFLICT (header_id, log_id) DO UPDATE SET header_id = $1, log_id = $2, variable1 = $3;`
+			Expect(actualQuery).To(Equal(expectedQuery))
+		})
+
+		It("marks log transformed", func() {
+			createErr := event.Create([]event.InsertionModel{testModel}, db)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var logTransformed bool
+			getErr := db.Get(&logTransformed, `SELECT transformed FROM public.header_sync_logs WHERE id = $1`, logID)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(logTransformed).To(BeTrue())
+		})
+	})
+})
+
+type TestEvent struct {
+	LogID     string `db:"log_id"`
+	Variable1 string
+}

libraries/shared/factories/event/transformer.go

@@ -17,10 +17,7 @@
 package event
 
 import (
-	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/sirupsen/logrus"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -33,38 +30,28 @@ type Transformer struct {
 }
 
 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.EventTransformer {
+	transformer.Converter.SetDB(db)
 	transformer.Repository.SetDB(db)
 	return transformer
 }
 
-func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+func (transformer Transformer) Execute(logs []core.HeaderSyncLog) error {
 	transformerName := transformer.Config.TransformerName
 	config := transformer.Config
 
 	if len(logs) < 1 {
-		err := transformer.Repository.MarkHeaderChecked(header.Id)
-		if err != nil {
-			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
-			return err
-		}
 		return nil
 	}
 
-	entities, err := transformer.Converter.ToEntities(config.ContractAbi, logs)
+	models, err := transformer.Converter.ToModels(config.ContractAbi, logs)
 	if err != nil {
-		log.Printf("Error converting logs to entities in %v: %v", transformerName, err)
+		logrus.Errorf("error converting entities to models in %v: %v", transformerName, err)
 		return err
 	}
 
-	models, err := transformer.Converter.ToModels(entities)
+	err = transformer.Repository.Create(models)
 	if err != nil {
-		log.Printf("Error converting entities to models in %v: %v", transformerName, err)
+		logrus.Errorf("error persisting %v record: %v", transformerName, err)
-		return err
-	}
-
-	err = transformer.Repository.Create(header.Id, models)
-	if err != nil {
-		log.Printf("Error persisting %v record: %v", transformerName, err)
 		return err
 	}
 

libraries/shared/factories/event/transformer_test.go

@@ -17,33 +17,29 @@
 package event_test
 
 import (
-	"math/rand"
-
-	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"math/rand"
 )
 
 var _ = Describe("Transformer", func() {
 	var (
-		repository mocks.MockRepository
+		repository mocks.MockEventRepository
 		converter  mocks.MockConverter
 		t          transformer.EventTransformer
 		headerOne  core.Header
 		config     = test_data.GenericTestConfig
-		logs       = test_data.GenericTestLogs
+		logs       []core.HeaderSyncLog
 	)
 
 	BeforeEach(func() {
-		repository = mocks.MockRepository{}
+		repository = mocks.MockEventRepository{}
 		converter = mocks.MockConverter{}
 
 		t = event.Transformer{
@@ -53,93 +49,56 @@ var _ = Describe("Transformer", func() {
 		}.NewTransformer(nil)
 
 		headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
+
+		logs = []core.HeaderSyncLog{{
+			ID:          0,
+			HeaderID:    headerOne.Id,
+			Log:         test_data.GenericTestLog(),
+			Transformed: false,
+		}}
 	})
 
 	It("sets the db", func() {
 		Expect(repository.SetDbCalled).To(BeTrue())
 	})
 
-	It("marks header checked if no logs returned", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
-	})
-
 	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]core.HeaderSyncLog{})
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
 		Expect(converter.ToModelsCalledCounter).To(Equal(0))
 		Expect(repository.CreateCalledCounter).To(Equal(0))
 	})
 
-	It("does not call repository.MarkCheckedHeader when there are logs", func() {
+	It("converts an eth log to a model", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs)
-
-		Expect(err).NotTo(HaveOccurred())
-		repository.AssertMarkHeaderCheckedNotCalled()
-	})
-
-	It("returns error if marking header checked returns err", func() {
-		repository.SetMarkHeaderCheckedError(fakes.FakeError)
-
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("converts an eth log to an entity", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
 		Expect(converter.LogsToConvert).To(Equal(logs))
 	})
 
-	It("returns an error if converter fails", func() {
-		converter.ToEntitiesError = fakes.FakeError
-
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("converts an entity to a model", func() {
-		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
-
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
-	})
-
 	It("returns an error if converting to models fails", func() {
-		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
 		converter.ToModelsError = fakes.FakeError
 
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs)
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
 	It("persists the record", func() {
-		converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
+		converter.ModelsToReturn = []event.InsertionModel{test_data.GenericModel}
 
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs)
 
 		Expect(err).NotTo(HaveOccurred())
-		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
+		Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel))
-		Expect(repository.PassedModels[0]).To(Equal(test_data.GenericModel{}))
 	})
 
 	It("returns error if persisting the record fails", func() {
 		repository.SetCreateError(fakes.FakeError)
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs)
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))

libraries/shared/factories/storage/README.md

@@ -31,15 +31,15 @@ The storage transformer depends on contract-specific implementations of code cap
 
 ```golang
 func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
-	value, decodeErr := shared.Decode(row, metadata)
+	value, decodeErr := utils.Decode(diff, metadata)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
 }
 ```
 
@@ -47,20 +47,36 @@ func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
 
 In order to watch an additional smart contract, a developer must create three things:
 
-1. Mappings - specify how to identify keys in the contract's storage trie.
+1. StorageKeysLoader - identify keys in the contract's storage trie, providing metadata to describe how associated values should be decoded.
 1. Repository - specify how to persist a parsed version of the storage value matching the recognized storage key.
 1. Instance - create an instance of the storage transformer that uses your mappings and repository.
 
-### Mappings
+### StorageKeysLoader
+
+A `StorageKeysLoader` is used by the `StorageKeysLookup` object on a storage transformer.
 
 ```golang
-type Mappings interface {
+type KeysLoader interface {
-	Lookup(key common.Hash) (shared.StorageValueMetadata, error)
+	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
 	SetDB(db *postgres.DB)
 }
 ```
 
-A contract-specific implementation of the mappings interface enables the storage transformer to fetch metadata associated with a storage key.
+When a key is not found, the lookup object refreshes its known keys by calling the loader.
+
+```golang
+func (lookup *keysLookup) refreshMappings() error {
+	var err error
+	lookup.mappings, err = lookup.loader.LoadMappings()
+	if err != nil {
+		return err
+	}
+	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
+	return nil
+}
+```
+
+A contract-specific implementation of the loader enables the storage transformer to fetch metadata associated with a storage key.
 
 Storage metadata contains: the name of the variable matching the storage key, a raw version of any keys associated with the variable (if the variable is a mapping), and the variable's type.
 
@@ -72,7 +88,7 @@ type StorageValueMetadata struct {
 }
 ```
 
-Keys are only relevant if the variable is a mapping. For example, in the following Solidity code:
+The `Keys` field on the metadata is only relevant if the variable is a mapping. For example, in the following Solidity code:
 
 ```solidity
 pragma solidity ^0.4.0;
@@ -85,7 +101,7 @@ contract Contract {
 
 The metadata for variable `x` would not have any associated keys, but the metadata for a storage key associated with `y` would include the address used to specify that key's index in the mapping.
 
-The `SetDB` function is required for the mappings to connect to the database.
+The `SetDB` function is required for the storage key loader to connect to the database.
 A database connection may be desired when keys in a mapping variable need to be read from log events (e.g. to lookup what addresses may exist in `y`, above).
 
 ### Repository
@@ -100,6 +116,7 @@ type Repository interface {
 A contract-specific implementation of the repository interface enables the transformer to write the decoded storage value to the appropriate table in postgres.
 
 The `Create` function is expected to recognize and persist a given storage value by the variable's name, as indicated on the row's metadata.
+Note: we advise silently discarding duplicates in `Create` - as it's possible that you may read the same diff several times, and an error will trigger the storage watcher to queue that diff for later processing.
 
 The `SetDB` function is required for the repository to connect to the database.
 

libraries/shared/factories/storage/keys_loader.go

@@ -0,0 +1,28 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type KeysLoader interface {
+	LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}

libraries/shared/factories/storage/keys_lookup.go

@@ -0,0 +1,66 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type KeysLookup interface {
+	Lookup(key common.Hash) (utils.StorageValueMetadata, error)
+	SetDB(db *postgres.DB)
+}
+
+type keysLookup struct {
+	loader   KeysLoader
+	mappings map[common.Hash]utils.StorageValueMetadata
+}
+
+func NewKeysLookup(loader KeysLoader) KeysLookup {
+	return &keysLookup{loader: loader, mappings: make(map[common.Hash]utils.StorageValueMetadata)}
+}
+
+func (lookup *keysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
+	metadata, ok := lookup.mappings[key]
+	if !ok {
+		refreshErr := lookup.refreshMappings()
+		if refreshErr != nil {
+			return metadata, refreshErr
+		}
+		metadata, ok = lookup.mappings[key]
+		if !ok {
+			return metadata, utils.ErrStorageKeyNotFound{Key: key.Hex()}
+		}
+	}
+	return metadata, nil
+}
+
+func (lookup *keysLookup) refreshMappings() error {
+	var err error
+	lookup.mappings, err = lookup.loader.LoadMappings()
+	if err != nil {
+		return err
+	}
+	lookup.mappings = utils.AddHashedKeys(lookup.mappings)
+	return nil
+}
+
+func (lookup *keysLookup) SetDB(db *postgres.DB) {
+	lookup.loader.SetDB(db)
+}

libraries/shared/factories/storage/keys_lookup_test.go

@@ -0,0 +1,113 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("Storage keys lookup", func() {
+	var (
+		fakeMetadata = utils.GetStorageValueMetadata("name", map[utils.Key]string{}, utils.Uint256)
+		lookup       storage.KeysLookup
+		loader       *mocks.MockStorageKeysLoader
+	)
+
+	BeforeEach(func() {
+		loader = &mocks.MockStorageKeysLoader{}
+		lookup = storage.NewKeysLookup(loader)
+	})
+
+	Describe("Lookup", func() {
+		Describe("when key not found", func() {
+			It("refreshes keys", func() {
+				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+
+			It("returns error if refreshing keys fails", func() {
+				loader.LoadMappingsError = fakes.FakeError
+
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+		})
+
+		Describe("when key found", func() {
+			BeforeEach(func() {
+				loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+				_, err := lookup.Lookup(fakes.FakeHash)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+
+			It("does not refresh keys", func() {
+				_, err := lookup.Lookup(fakes.FakeHash)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(loader.LoadMappingsCallCount).To(Equal(1))
+			})
+		})
+
+		It("returns metadata for loaded static key", func() {
+			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+
+			metadata, err := lookup.Lookup(fakes.FakeHash)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(metadata).To(Equal(fakeMetadata))
+		})
+
+		It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() {
+			loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
+
+			hashedKey := common.BytesToHash(crypto.Keccak256(fakes.FakeHash.Bytes()))
+			metadata, err := lookup.Lookup(hashedKey)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(metadata).To(Equal(fakeMetadata))
+		})
+
+		It("returns key not found error if key not found", func() {
+			_, err := lookup.Lookup(fakes.FakeHash)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(utils.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
+		})
+	})
+
+	Describe("SetDB", func() {
+		It("sets the db on the loader", func() {
+			lookup.SetDB(test_config.NewTestDB(test_config.NewTestNode()))
+
+			Expect(loader.SetDBCalled).To(BeTrue())
+		})
+	})
+})

libraries/shared/factories/storage/repository.go

@@ -14,14 +14,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package repository
+package storage
 
 import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-type StorageRepository interface {
+type Repository interface {
 	Create(blockNumber int, blockHash string, metadata utils.StorageValueMetadata, value interface{}) error
 	SetDB(db *postgres.DB)
 }

libraries/shared/factories/storage/transformer.go

@@ -18,38 +18,35 @@ package storage
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type Transformer struct {
-	Address    common.Address
+	HashedAddress     common.Hash
-	Mappings   storage.Mappings
+	StorageKeysLookup KeysLookup
-	Repository repository.StorageRepository
+	Repository        Repository
 }
 
 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
-	transformer.Mappings.SetDB(db)
+	transformer.StorageKeysLookup.SetDB(db)
 	transformer.Repository.SetDB(db)
 	return transformer
 }
 
-func (transformer Transformer) ContractAddress() common.Address {
+func (transformer Transformer) KeccakContractAddress() common.Hash {
-	return transformer.Address
+	return transformer.HashedAddress
 }
 
-func (transformer Transformer) Execute(row utils.StorageDiffRow) error {
+func (transformer Transformer) Execute(diff utils.StorageDiff) error {
-	metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
+	metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
 	if lookupErr != nil {
 		return lookupErr
 	}
-	value, decodeErr := utils.Decode(row, metadata)
+	value, decodeErr := utils.Decode(diff, metadata)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
+	return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
 }

libraries/shared/factories/storage/transformer_test.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
@@ -29,38 +28,38 @@ import (
 
 var _ = Describe("Storage transformer", func() {
 	var (
-		mappings   *mocks.MockMappings
+		storageKeysLookup *mocks.MockStorageKeysLookup
-		repository *mocks.MockStorageRepository
+		repository        *mocks.MockStorageRepository
-		t          storage.Transformer
+		t                 storage.Transformer
 	)
 
 	BeforeEach(func() {
-		mappings = &mocks.MockMappings{}
+		storageKeysLookup = &mocks.MockStorageKeysLookup{}
 		repository = &mocks.MockStorageRepository{}
 		t = storage.Transformer{
-			Address:    common.Address{},
+			HashedAddress:     common.Hash{},
-			Mappings:   mappings,
+			StorageKeysLookup: storageKeysLookup,
-			Repository: repository,
+			Repository:        repository,
 		}
 	})
 
 	It("returns the contract address being watched", func() {
-		fakeAddress := common.HexToAddress("0x12345")
+		fakeAddress := utils.HexToKeccak256Hash("0x12345")
-		t.Address = fakeAddress
+		t.HashedAddress = fakeAddress
 
-		Expect(t.ContractAddress()).To(Equal(fakeAddress))
+		Expect(t.KeccakContractAddress()).To(Equal(fakeAddress))
 	})
 
 	It("looks up metadata for storage key", func() {
-		t.Execute(utils.StorageDiffRow{})
+		t.Execute(utils.StorageDiff{})
 
-		Expect(mappings.LookupCalled).To(BeTrue())
+		Expect(storageKeysLookup.LookupCalled).To(BeTrue())
 	})
 
 	It("returns error if lookup fails", func() {
-		mappings.LookupErr = fakes.FakeError
+		storageKeysLookup.LookupErr = fakes.FakeError
 
-		err := t.Execute(utils.StorageDiffRow{})
+		err := t.Execute(utils.StorageDiff{})
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -68,16 +67,16 @@ var _ = Describe("Storage transformer", func() {
 
 	It("creates storage row with decoded data", func() {
 		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		mappings.Metadata = fakeMetadata
+		storageKeysLookup.Metadata = fakeMetadata
 		rawValue := common.HexToAddress("0x12345")
 		fakeBlockNumber := 123
 		fakeBlockHash := "0x67890"
-		fakeRow := utils.StorageDiffRow{
+		fakeRow := utils.StorageDiff{
-			Contract:     common.Address{},
+			HashedAddress: common.Hash{},
-			BlockHash:    common.HexToHash(fakeBlockHash),
+			BlockHash:     common.HexToHash(fakeBlockHash),
-			BlockHeight:  fakeBlockNumber,
+			BlockHeight:   fakeBlockNumber,
-			StorageKey:   common.Hash{},
+			StorageKey:    common.Hash{},
-			StorageValue: rawValue.Hash(),
+			StorageValue:  rawValue.Hash(),
 		}
 
 		err := t.Execute(fakeRow)
@@ -92,12 +91,62 @@ var _ = Describe("Storage transformer", func() {
 	It("returns error if creating row fails", func() {
 		rawValue := common.HexToAddress("0x12345")
 		fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
-		mappings.Metadata = fakeMetadata
+		storageKeysLookup.Metadata = fakeMetadata
 		repository.CreateErr = fakes.FakeError
 
-		err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+		err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
+
+	Describe("when a storage row contains more than one item packed in storage", func() {
+		var (
+			rawValue        = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
+			fakeBlockNumber = 123
+			fakeBlockHash   = "0x67890"
+			packedTypes     = make(map[int]utils.ValueType)
+		)
+		packedTypes[0] = utils.Uint48
+		packedTypes[1] = utils.Uint48
+
+		var fakeMetadata = utils.StorageValueMetadata{
+			Name:        "",
+			Keys:        nil,
+			Type:        utils.PackedSlot,
+			PackedTypes: packedTypes,
+		}
+
+		It("passes the decoded data items to the repository", func() {
+			storageKeysLookup.Metadata = fakeMetadata
+			fakeRow := utils.StorageDiff{
+				HashedAddress: common.Hash{},
+				BlockHash:     common.HexToHash(fakeBlockHash),
+				BlockHeight:   fakeBlockNumber,
+				StorageKey:    common.Hash{},
+				StorageValue:  rawValue.Hash(),
+			}
+
+			err := t.Execute(fakeRow)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
+			Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
+			Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
+			expectedPassedValue := make(map[int]string)
+			expectedPassedValue[0] = "10800"
+			expectedPassedValue[1] = "172800"
+			Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
+		})
+
+		It("returns error if creating a row fails", func() {
+			storageKeysLookup.Metadata = fakeMetadata
+			repository.CreateErr = fakes.FakeError
+
+			err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
 })

libraries/shared/fetcher/csv_tail_storage_fetcher.go

@@ -17,15 +17,12 @@
 package fetcher
 
 import (
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	"strings"
 )
 
-type IStorageFetcher interface {
-	FetchStorageDiffs(chan<- utils.StorageDiffRow, chan<- error)
-}
-
 type CsvTailStorageFetcher struct {
 	tailer fs.Tailer
 }
@@ -34,17 +31,18 @@ func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
 	return CsvTailStorageFetcher{tailer: tailer}
 }
 
-func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	t, tailErr := storageFetcher.tailer.Tail()
 	if tailErr != nil {
 		errs <- tailErr
 	}
+	logrus.Debug("fetching storage diffs...")
 	for line := range t.Lines {
-		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+		diff, parseErr := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		if parseErr != nil {
 			errs <- parseErr
 		} else {
-			out <- row
+			out <- diff
 		}
 	}
 }

libraries/shared/fetcher/csv_tail_storage_fetcher_test.go

@@ -18,30 +18,28 @@ package fetcher_test
 
 import (
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/hpcloud/tail"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"strings"
+	"time"
 )
 
 var _ = Describe("Csv Tail Storage Fetcher", func() {
 	var (
 		errorsChannel  chan error
 		mockTailer     *fakes.MockTailer
-		rowsChannel    chan utils.StorageDiffRow
+		diffsChannel   chan utils.StorageDiff
 		storageFetcher fetcher.CsvTailStorageFetcher
 	)
 
 	BeforeEach(func() {
 		errorsChannel = make(chan error)
-		rowsChannel = make(chan utils.StorageDiffRow)
+		diffsChannel = make(chan utils.StorageDiff)
 		mockTailer = fakes.NewMockTailer()
 		storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
 	})
@@ -49,7 +47,7 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 	It("adds error to errors channel if tailing file fails", func(done Done) {
 		mockTailer.TailErr = fakes.FakeError
 
-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 
 		Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
 		close(done)
@@ -58,24 +56,24 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 	It("adds parsed csv row to rows channel for storage diff", func(done Done) {
 		line := getFakeLine()
 
-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line
 
-		expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
+		expectedRow, err := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		Expect(err).NotTo(HaveOccurred())
-		Expect(<-rowsChannel).To(Equal(expectedRow))
+		Expect(<-diffsChannel).To(Equal(expectedRow))
 		close(done)
 	})
 
 	It("adds error to errors channel if parsing csv fails", func(done Done) {
 		line := &tail.Line{Text: "invalid"}
 
-		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line
 
 		Expect(<-errorsChannel).To(HaveOccurred())
 		select {
-		case <-rowsChannel:
+		case <-diffsChannel:
 			Fail("value passed to rows channel on error")
 		default:
 			Succeed()

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -0,0 +1,81 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher
+
+import (
+	"fmt"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+)
+
+type GethRpcStorageFetcher struct {
+	statediffPayloadChan chan statediff.Payload
+	streamer             streamer.Streamer
+}
+
+func NewGethRpcStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan statediff.Payload) GethRpcStorageFetcher {
+	return GethRpcStorageFetcher{
+		statediffPayloadChan: statediffPayloadChan,
+		streamer:             streamer,
+	}
+}
+
+func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
+	ethStatediffPayloadChan := fetcher.statediffPayloadChan
+	clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
+	if clientSubErr != nil {
+		errs <- clientSubErr
+		panic(fmt.Sprintf("Error creating a geth client subscription: %v", clientSubErr))
+	}
+	logrus.Info("Successfully created a geth client subscription: ", clientSubscription)
+
+	for {
+		diff := <-ethStatediffPayloadChan
+		logrus.Trace("received a statediff")
+		stateDiff := new(statediff.StateDiff)
+		decodeErr := rlp.DecodeBytes(diff.StateDiffRlp, stateDiff)
+		if decodeErr != nil {
+			logrus.Warn("Error decoding state diff into RLP: ", decodeErr)
+			errs <- decodeErr
+		}
+
+		accounts := getAccountsFromDiff(*stateDiff)
+		logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
+		for _, account := range accounts {
+			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
+			for _, storage := range account.Storage {
+				diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
+				logrus.Trace("adding storage diff to out channel",
+					"keccak of address: ", diff.HashedAddress.Hex(),
+					"block height: ", diff.BlockHeight,
+					"storage key: ", diff.StorageKey.Hex(),
+					"storage value: ", diff.StorageValue.Hex())
+				if formatErr != nil {
+					errs <- formatErr
+				}
+
+				out <- diff
+			}
+		}
+	}
+}
+
+func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
+	accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
+	return append(accounts, stateDiff.DeletedAccounts...)
+}

libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go

@@ -0,0 +1,172 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetcher_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+type MockStoragediffStreamer struct {
+	subscribeError    error
+	PassedPayloadChan chan statediff.Payload
+	streamPayloads    []statediff.Payload
+}
+
+func (streamer *MockStoragediffStreamer) Stream(statediffPayloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	clientSubscription := rpc.ClientSubscription{}
+	streamer.PassedPayloadChan = statediffPayloadChan
+
+	go func() {
+		for _, payload := range streamer.streamPayloads {
+			streamer.PassedPayloadChan <- payload
+		}
+	}()
+
+	return &clientSubscription, streamer.subscribeError
+}
+
+func (streamer *MockStoragediffStreamer) SetSubscribeError(err error) {
+	streamer.subscribeError = err
+}
+
+func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payload) {
+	streamer.streamPayloads = payloads
+}
+
+var _ = Describe("Geth RPC Storage Fetcher", func() {
+	var streamer MockStoragediffStreamer
+	var statediffPayloadChan chan statediff.Payload
+	var statediffFetcher fetcher.GethRpcStorageFetcher
+	var storagediffChan chan utils.StorageDiff
+	var errorChan chan error
+
+	BeforeEach(func() {
+		streamer = MockStoragediffStreamer{}
+		statediffPayloadChan = make(chan statediff.Payload, 1)
+		statediffFetcher = fetcher.NewGethRpcStorageFetcher(&streamer, statediffPayloadChan)
+		storagediffChan = make(chan utils.StorageDiff)
+		errorChan = make(chan error)
+	})
+
+	It("adds errors to error channel if the RPC subscription fails and panics", func(done Done) {
+		streamer.SetSubscribeError(fakes.FakeError)
+
+		go func() {
+			failedSub := func() {
+				statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+			}
+			Expect(failedSub).To(Panic())
+		}()
+
+		Expect(<-errorChan).To(MatchError(fakes.FakeError))
+		close(done)
+	})
+
+	It("streams StatediffPayloads from a Geth RPC subscription", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		streamedPayload := <-statediffPayloadChan
+		Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload))
+		Expect(streamer.PassedPayloadChan).To(Equal(statediffPayloadChan))
+		close(done)
+	})
+
+	It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) {
+		badStatediffPayload := statediff.Payload{}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("EOF"))
+
+		close(done)
+	})
+
+	It("adds parsed statediff payloads to the rows channel", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		height := test_data.BlockNumber
+		intHeight := int(height.Int64())
+		createdExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
+		}
+		updatedExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.LargeStorageValue),
+		}
+		deletedExpectedStorageDiff := utils.StorageDiff{
+			HashedAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:   intHeight,
+			StorageKey:    common.BytesToHash(test_data.StorageKey),
+			StorageValue:  common.BytesToHash(test_data.SmallStorageValue),
+		}
+
+		createdStateDiff := <-storagediffChan
+		updatedStateDiff := <-storagediffChan
+		deletedStateDiff := <-storagediffChan
+
+		Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff))
+		Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff))
+		Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff))
+
+		close(done)
+	})
+
+	It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) {
+		accountDiffs := test_data.CreatedAccountDiffs
+		accountDiffs[0].Storage = []statediff.StorageDiff{test_data.StorageWithBadValue}
+
+		stateDiff := statediff.StateDiff{
+			BlockNumber:     test_data.BlockNumber,
+			BlockHash:       common.HexToHash(test_data.BlockHash),
+			CreatedAccounts: accountDiffs,
+		}
+
+		stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+		Expect(err).NotTo(HaveOccurred())
+
+		badStatediffPayload := statediff.Payload{
+			StateDiffRlp: stateDiffRlp,
+		}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("rlp: input contains more than one value"))
+
+		close(done)
+	})
+})

libraries/shared/fetcher/log_fetcher.go

@@ -26,6 +26,7 @@ import (
 
 type ILogFetcher interface {
 	FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
+	// TODO Extend FetchLogs for doing several blocks at a time
 }
 
 type LogFetcher struct {

libraries/shared/fetcher/storage_fetcher_interface.go

@@ -1,10 +1,10 @@
-// Copyright © 2017 NAME HERE <EMAIL ADDRESS>
+// Copyright 2018 Vulcanize
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package fetcher
 
-import "github.com/spf13/testproject/cmd"
+import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 
-func main() {
+type IStorageFetcher interface {
-	cmd.Execute()
+	FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error)
 }

libraries/shared/logs/delegator.go

@@ -0,0 +1,85 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs
+
+import (
+	"errors"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+var (
+	ErrNoLogs         = errors.New("no logs available for transforming")
+	ErrNoTransformers = errors.New("no event transformers configured in the log delegator")
+)
+
+type ILogDelegator interface {
+	AddTransformer(t transformer.EventTransformer)
+	DelegateLogs() error
+}
+
+type LogDelegator struct {
+	Chunker       chunker.Chunker
+	LogRepository datastore.HeaderSyncLogRepository
+	Transformers  []transformer.EventTransformer
+}
+
+func (delegator *LogDelegator) AddTransformer(t transformer.EventTransformer) {
+	delegator.Transformers = append(delegator.Transformers, t)
+	delegator.Chunker.AddConfig(t.GetConfig())
+}
+
+func (delegator *LogDelegator) DelegateLogs() error {
+	if len(delegator.Transformers) < 1 {
+		return ErrNoTransformers
+	}
+
+	persistedLogs, fetchErr := delegator.LogRepository.GetUntransformedHeaderSyncLogs()
+	if fetchErr != nil {
+		logrus.Errorf("error loading logs from db: %s", fetchErr.Error())
+		return fetchErr
+	}
+
+	if len(persistedLogs) < 1 {
+		return ErrNoLogs
+	}
+
+	transformErr := delegator.delegateLogs(persistedLogs)
+	if transformErr != nil {
+		logrus.Errorf("error transforming logs: %s", transformErr)
+		return transformErr
+	}
+
+	return nil
+}
+
+func (delegator *LogDelegator) delegateLogs(logs []core.HeaderSyncLog) error {
+	chunkedLogs := delegator.Chunker.ChunkLogs(logs)
+	for _, t := range delegator.Transformers {
+		transformerName := t.GetConfig().TransformerName
+		logChunk := chunkedLogs[transformerName]
+		err := t.Execute(logChunk)
+		if err != nil {
+			logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
+			return err
+		}
+	}
+	return nil
+}

libraries/shared/logs/delegator_test.go

@@ -0,0 +1,165 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/chunker"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/logs"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"strings"
+)
+
+var _ = Describe("Log delegator", func() {
+	Describe("AddTransformer", func() {
+		It("adds transformers to the delegator", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			delegator := logs.LogDelegator{Chunker: chunker.NewLogChunker()}
+
+			delegator.AddTransformer(fakeTransformer)
+
+			Expect(delegator.Transformers).To(Equal([]transformer.EventTransformer{fakeTransformer}))
+		})
+
+		It("passes transformers' configs to the chunker", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			fakeConfig := mocks.FakeTransformerConfig
+			fakeTransformer.SetTransformerConfig(fakeConfig)
+			chunker := chunker.NewLogChunker()
+			delegator := logs.LogDelegator{Chunker: chunker}
+
+			delegator.AddTransformer(fakeTransformer)
+
+			expectedName := fakeConfig.TransformerName
+			expectedTopic := common.HexToHash(fakeConfig.Topic)
+			Expect(chunker.NameToTopic0).To(Equal(map[string]common.Hash{expectedName: expectedTopic}))
+			expectedAddress := strings.ToLower(fakeConfig.ContractAddresses[0])
+			Expect(chunker.AddressToNames).To(Equal(map[string][]string{expectedAddress: {expectedName}}))
+		})
+	})
+
+	Describe("DelegateLogs", func() {
+		It("returns error if no transformers configured", func() {
+			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(logs.ErrNoTransformers))
+		})
+
+		It("gets untransformed logs", func() {
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(&mocks.MockEventTransformer{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mockLogRepository.GetCalled).To(BeTrue())
+		})
+
+		It("returns error if getting untransformed logs fails", func() {
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.GetError = fakes.FakeError
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(&mocks.MockEventTransformer{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("returns error that no logs were found if no logs returned", func() {
+			delegator := newDelegator(&fakes.MockHeaderSyncLogRepository{})
+			delegator.AddTransformer(&mocks.MockEventTransformer{})
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(logs.ErrNoLogs))
+		})
+
+		It("delegates chunked logs to transformers", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			config := mocks.FakeTransformerConfig
+			fakeTransformer.SetTransformerConfig(config)
+			fakeGethLog := types.Log{
+				Address: common.HexToAddress(config.ContractAddresses[0]),
+				Topics:  []common.Hash{common.HexToHash(config.Topic)},
+			}
+			fakeHeaderSyncLogs := []core.HeaderSyncLog{{Log: fakeGethLog}}
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(fakeTransformer)
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(fakeTransformer.ExecuteWasCalled).To(BeTrue())
+			Expect(fakeTransformer.PassedLogs).To(Equal(fakeHeaderSyncLogs))
+		})
+
+		It("returns error if transformer returns an error", func() {
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = []core.HeaderSyncLog{{}}
+			delegator := newDelegator(mockLogRepository)
+			fakeTransformer := &mocks.MockEventTransformer{ExecuteError: fakes.FakeError}
+			delegator.AddTransformer(fakeTransformer)
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		It("returns nil for error when logs returned and delegated", func() {
+			fakeTransformer := &mocks.MockEventTransformer{}
+			config := mocks.FakeTransformerConfig
+			fakeTransformer.SetTransformerConfig(config)
+			fakeGethLog := types.Log{
+				Address: common.HexToAddress(config.ContractAddresses[0]),
+				Topics:  []common.Hash{common.HexToHash(config.Topic)},
+			}
+			fakeHeaderSyncLogs := []core.HeaderSyncLog{{Log: fakeGethLog}}
+			mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+			mockLogRepository.ReturnLogs = fakeHeaderSyncLogs
+			delegator := newDelegator(mockLogRepository)
+			delegator.AddTransformer(fakeTransformer)
+
+			err := delegator.DelegateLogs()
+
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})
+
+func newDelegator(headerSyncLogRepository *fakes.MockHeaderSyncLogRepository) *logs.LogDelegator {
+	return &logs.LogDelegator{
+		Chunker:       chunker.NewLogChunker(),
+		LogRepository: headerSyncLogRepository,
+	}
+}

libraries/shared/logs/extractor.go

@@ -0,0 +1,154 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs
+
+import (
+	"errors"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/sirupsen/logrus"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transactions"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore"
+)
+
+var (
+	ErrNoUncheckedHeaders = errors.New("no unchecked headers available for log fetching")
+	ErrNoWatchedAddresses = errors.New("no watched addresses configured in the log extractor")
+)
+
+type ILogExtractor interface {
+	AddTransformerConfig(config transformer.EventTransformerConfig) error
+	ExtractLogs(recheckHeaders constants.TransformerExecution) error
+}
+
+type LogExtractor struct {
+	Addresses                []common.Address
+	CheckedHeadersRepository datastore.CheckedHeadersRepository
+	CheckedLogsRepository    datastore.CheckedLogsRepository
+	Fetcher                  fetcher.ILogFetcher
+	LogRepository            datastore.HeaderSyncLogRepository
+	StartingBlock            *int64
+	Syncer                   transactions.ITransactionsSyncer
+	Topics                   []common.Hash
+}
+
+// Add additional logs to extract
+func (extractor *LogExtractor) AddTransformerConfig(config transformer.EventTransformerConfig) error {
+	checkedHeadersErr := extractor.updateCheckedHeaders(config)
+	if checkedHeadersErr != nil {
+		return checkedHeadersErr
+	}
+
+	if extractor.StartingBlock == nil {
+		extractor.StartingBlock = &config.StartingBlockNumber
+	} else if earlierStartingBlockNumber(config.StartingBlockNumber, *extractor.StartingBlock) {
+		extractor.StartingBlock = &config.StartingBlockNumber
+	}
+
+	addresses := transformer.HexStringsToAddresses(config.ContractAddresses)
+	extractor.Addresses = append(extractor.Addresses, addresses...)
+	extractor.Topics = append(extractor.Topics, common.HexToHash(config.Topic))
+	return nil
+}
+
+// Fetch and persist watched logs
+func (extractor LogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
+	if len(extractor.Addresses) < 1 {
+		logrus.Errorf("error extracting logs: %s", ErrNoWatchedAddresses.Error())
+		return ErrNoWatchedAddresses
+	}
+
+	uncheckedHeaders, uncheckedHeadersErr := extractor.CheckedHeadersRepository.UncheckedHeaders(*extractor.StartingBlock, -1, getCheckCount(recheckHeaders))
+	if uncheckedHeadersErr != nil {
+		logrus.Errorf("error fetching missing headers: %s", uncheckedHeadersErr)
+		return uncheckedHeadersErr
+	}
+
+	if len(uncheckedHeaders) < 1 {
+		return ErrNoUncheckedHeaders
+	}
+
+	for _, header := range uncheckedHeaders {
+		logs, fetchLogsErr := extractor.Fetcher.FetchLogs(extractor.Addresses, extractor.Topics, header)
+		if fetchLogsErr != nil {
+			logError("error fetching logs for header: %s", fetchLogsErr, header)
+			return fetchLogsErr
+		}
+
+		if len(logs) > 0 {
+			transactionsSyncErr := extractor.Syncer.SyncTransactions(header.Id, logs)
+			if transactionsSyncErr != nil {
+				logError("error syncing transactions: %s", transactionsSyncErr, header)
+				return transactionsSyncErr
+			}
+
+			createLogsErr := extractor.LogRepository.CreateHeaderSyncLogs(header.Id, logs)
+			if createLogsErr != nil {
+				logError("error persisting logs: %s", createLogsErr, header)
+				return createLogsErr
+			}
+		}
+
+		markHeaderCheckedErr := extractor.CheckedHeadersRepository.MarkHeaderChecked(header.Id)
+		if markHeaderCheckedErr != nil {
+			logError("error marking header checked: %s", markHeaderCheckedErr, header)
+			return markHeaderCheckedErr
+		}
+	}
+	return nil
+}
+
+func earlierStartingBlockNumber(transformerBlock, watcherBlock int64) bool {
+	return transformerBlock < watcherBlock
+}
+
+func logError(description string, err error, header core.Header) {
+	logrus.WithFields(logrus.Fields{
+		"headerId":    header.Id,
+		"headerHash":  header.Hash,
+		"blockNumber": header.BlockNumber,
+	}).Errorf(description, err.Error())
+}
+
+func getCheckCount(recheckHeaders constants.TransformerExecution) int64 {
+	if recheckHeaders == constants.HeaderUnchecked {
+		return 1
+	} else {
+		return constants.RecheckHeaderCap
+	}
+}
+
+func (extractor *LogExtractor) updateCheckedHeaders(config transformer.EventTransformerConfig) error {
+	alreadyWatchingLog, watchingLogErr := extractor.CheckedLogsRepository.AlreadyWatchingLog(config.ContractAddresses, config.Topic)
+	if watchingLogErr != nil {
+		return watchingLogErr
+	}
+	if !alreadyWatchingLog {
+		uncheckHeadersErr := extractor.CheckedHeadersRepository.MarkHeadersUnchecked(config.StartingBlockNumber)
+		if uncheckHeadersErr != nil {
+			return uncheckHeadersErr
+		}
+		markLogWatchedErr := extractor.CheckedLogsRepository.MarkLogWatched(config.ContractAddresses, config.Topic)
+		if markLogWatchedErr != nil {
+			return markLogWatchedErr
+		}
+	}
+	return nil
+}

libraries/shared/logs/extractor_test.go

@@ -0,0 +1,414 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/logs"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"math/rand"
+)
+
+var _ = Describe("Log extractor", func() {
+	var (
+		checkedHeadersRepository *fakes.MockCheckedHeadersRepository
+		checkedLogsRepository    *fakes.MockCheckedLogsRepository
+		extractor                *logs.LogExtractor
+	)
+
+	BeforeEach(func() {
+		checkedHeadersRepository = &fakes.MockCheckedHeadersRepository{}
+		checkedLogsRepository = &fakes.MockCheckedLogsRepository{}
+		extractor = &logs.LogExtractor{
+			CheckedHeadersRepository: checkedHeadersRepository,
+			CheckedLogsRepository:    checkedLogsRepository,
+			Fetcher:                  &mocks.MockLogFetcher{},
+			LogRepository:            &fakes.MockHeaderSyncLogRepository{},
+			Syncer:                   &fakes.MockTransactionSyncer{},
+		}
+	})
+
+	Describe("AddTransformerConfig", func() {
+		It("updates extractor's starting block number to earliest available", func() {
+			earlierStartingBlockNumber := rand.Int63()
+			laterStartingBlockNumber := earlierStartingBlockNumber + 1
+
+			errOne := extractor.AddTransformerConfig(getTransformerConfig(laterStartingBlockNumber))
+			Expect(errOne).NotTo(HaveOccurred())
+			errTwo := extractor.AddTransformerConfig(getTransformerConfig(earlierStartingBlockNumber))
+			Expect(errTwo).NotTo(HaveOccurred())
+
+			Expect(*extractor.StartingBlock).To(Equal(earlierStartingBlockNumber))
+		})
+
+		It("adds transformer's addresses to extractor's watched addresses", func() {
+			addresses := []string{"0xA", "0xB"}
+			configWithAddresses := transformer.EventTransformerConfig{
+				ContractAddresses:   addresses,
+				StartingBlockNumber: rand.Int63(),
+			}
+
+			err := extractor.AddTransformerConfig(configWithAddresses)
+
+			Expect(err).NotTo(HaveOccurred())
+			expectedAddresses := transformer.HexStringsToAddresses(addresses)
+			Expect(extractor.Addresses).To(Equal(expectedAddresses))
+		})
+
+		It("adds transformer's topic to extractor's watched topics", func() {
+			topic := "0x1"
+			configWithTopic := transformer.EventTransformerConfig{
+				ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+				Topic:               topic,
+				StartingBlockNumber: rand.Int63(),
+			}
+
+			err := extractor.AddTransformerConfig(configWithTopic)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(extractor.Topics).To(Equal([]common.Hash{common.HexToHash(topic)}))
+		})
+
+		It("returns error if checking whether log has been checked returns error", func() {
+			checkedLogsRepository.AlreadyWatchingLogError = fakes.FakeError
+
+			err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		Describe("when log has previously been checked", func() {
+			It("does not mark any headers unchecked", func() {
+				checkedLogsRepository.AlreadyWatchingLogReturn = true
+
+				err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(checkedHeadersRepository.MarkHeadersUncheckedCalled).To(BeFalse())
+			})
+		})
+
+		Describe("when log has not previously been checked", func() {
+			BeforeEach(func() {
+				checkedLogsRepository.AlreadyWatchingLogReturn = false
+			})
+
+			It("marks headers since transformer's starting block number as unchecked", func() {
+				blockNumber := rand.Int63()
+
+				err := extractor.AddTransformerConfig(getTransformerConfig(blockNumber))
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(checkedHeadersRepository.MarkHeadersUncheckedCalled).To(BeTrue())
+				Expect(checkedHeadersRepository.MarkHeadersUncheckedStartingBlockNumber).To(Equal(blockNumber))
+			})
+
+			It("returns error if marking headers unchecked returns error", func() {
+				checkedHeadersRepository.MarkHeadersUncheckedReturnError = fakes.FakeError
+
+				err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+
+			It("persists that tranformer's log has been checked", func() {
+				config := getTransformerConfig(rand.Int63())
+
+				err := extractor.AddTransformerConfig(config)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(checkedLogsRepository.MarkLogWatchedAddresses).To(Equal(config.ContractAddresses))
+				Expect(checkedLogsRepository.MarkLogWatchedTopicZero).To(Equal(config.Topic))
+			})
+
+			It("returns error if marking logs checked returns error", func() {
+				checkedLogsRepository.MarkLogWatchedError = fakes.FakeError
+
+				err := extractor.AddTransformerConfig(getTransformerConfig(rand.Int63()))
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+		})
+	})
+
+	Describe("ExtractLogs", func() {
+		It("returns error if no watched addresses configured", func() {
+			err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(logs.ErrNoWatchedAddresses))
+		})
+
+		Describe("when checking unchecked headers", func() {
+			It("gets headers since configured starting block with check_count < 1", func() {
+				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
+				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+				startingBlockNumber := rand.Int63()
+				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
+				Expect(mockCheckedHeadersRepository.UncheckedHeadersEndingBlockNumber).To(Equal(int64(-1)))
+				Expect(mockCheckedHeadersRepository.UncheckedHeadersCheckCount).To(Equal(int64(1)))
+			})
+		})
+
+		Describe("when rechecking headers", func() {
+			It("gets headers since configured starting block with check_count < RecheckHeaderCap", func() {
+				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
+				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+				startingBlockNumber := rand.Int63()
+				extractor.AddTransformerConfig(getTransformerConfig(startingBlockNumber))
+
+				err := extractor.ExtractLogs(constants.HeaderRecheck)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(mockCheckedHeadersRepository.UncheckedHeadersStartingBlockNumber).To(Equal(startingBlockNumber))
+				Expect(mockCheckedHeadersRepository.UncheckedHeadersEndingBlockNumber).To(Equal(int64(-1)))
+				Expect(mockCheckedHeadersRepository.UncheckedHeadersCheckCount).To(Equal(constants.RecheckHeaderCap))
+			})
+		})
+
+		It("returns error if getting unchecked headers fails", func() {
+			addTransformerConfig(extractor)
+			mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+			mockCheckedHeadersRepository.UncheckedHeadersReturnError = fakes.FakeError
+			extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+
+			err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+
+		Describe("when no unchecked headers", func() {
+			It("does not fetch logs", func() {
+				addTransformerConfig(extractor)
+				mockLogFetcher := &mocks.MockLogFetcher{}
+				extractor.Fetcher = mockLogFetcher
+
+				_ = extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(mockLogFetcher.FetchCalled).To(BeFalse())
+			})
+
+			It("returns error that no unchecked headers were found", func() {
+				addTransformerConfig(extractor)
+				mockLogFetcher := &mocks.MockLogFetcher{}
+				extractor.Fetcher = mockLogFetcher
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).To(MatchError(logs.ErrNoUncheckedHeaders))
+			})
+		})
+
+		Describe("when there are unchecked headers", func() {
+			It("fetches logs for unchecked headers", func() {
+				addUncheckedHeader(extractor)
+				config := transformer.EventTransformerConfig{
+					ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+					Topic:               fakes.FakeHash.Hex(),
+					StartingBlockNumber: rand.Int63(),
+				}
+				addTransformerErr := extractor.AddTransformerConfig(config)
+				Expect(addTransformerErr).NotTo(HaveOccurred())
+				mockLogFetcher := &mocks.MockLogFetcher{}
+				extractor.Fetcher = mockLogFetcher
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(mockLogFetcher.FetchCalled).To(BeTrue())
+				expectedTopics := []common.Hash{common.HexToHash(config.Topic)}
+				Expect(mockLogFetcher.Topics).To(Equal(expectedTopics))
+				expectedAddresses := transformer.HexStringsToAddresses(config.ContractAddresses)
+				Expect(mockLogFetcher.ContractAddresses).To(Equal(expectedAddresses))
+			})
+
+			It("returns error if fetching logs fails", func() {
+				addUncheckedHeader(extractor)
+				addTransformerConfig(extractor)
+				mockLogFetcher := &mocks.MockLogFetcher{}
+				mockLogFetcher.ReturnError = fakes.FakeError
+				extractor.Fetcher = mockLogFetcher
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+
+			Describe("when no fetched logs", func() {
+				It("does not sync transactions", func() {
+					addUncheckedHeader(extractor)
+					addTransformerConfig(extractor)
+					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
+					extractor.Syncer = mockTransactionSyncer
+
+					err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+					Expect(err).NotTo(HaveOccurred())
+					Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeFalse())
+				})
+			})
+
+			Describe("when there are fetched logs", func() {
+				It("syncs transactions", func() {
+					addUncheckedHeader(extractor)
+					addFetchedLog(extractor)
+					addTransformerConfig(extractor)
+					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
+					extractor.Syncer = mockTransactionSyncer
+
+					err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+					Expect(err).NotTo(HaveOccurred())
+					Expect(mockTransactionSyncer.SyncTransactionsCalled).To(BeTrue())
+				})
+
+				It("returns error if syncing transactions fails", func() {
+					addUncheckedHeader(extractor)
+					addFetchedLog(extractor)
+					addTransformerConfig(extractor)
+					mockTransactionSyncer := &fakes.MockTransactionSyncer{}
+					mockTransactionSyncer.SyncTransactionsError = fakes.FakeError
+					extractor.Syncer = mockTransactionSyncer
+
+					err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+					Expect(err).To(HaveOccurred())
+					Expect(err).To(MatchError(fakes.FakeError))
+				})
+
+				It("persists fetched logs", func() {
+					addUncheckedHeader(extractor)
+					addTransformerConfig(extractor)
+					fakeLogs := []types.Log{{
+						Address: common.HexToAddress("0xA"),
+						Topics:  []common.Hash{common.HexToHash("0xA")},
+						Data:    []byte{},
+						Index:   0,
+					}}
+					mockLogFetcher := &mocks.MockLogFetcher{ReturnLogs: fakeLogs}
+					extractor.Fetcher = mockLogFetcher
+					mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+					extractor.LogRepository = mockLogRepository
+
+					err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+					Expect(err).NotTo(HaveOccurred())
+					Expect(mockLogRepository.PassedLogs).To(Equal(fakeLogs))
+				})
+
+				It("returns error if persisting logs fails", func() {
+					addUncheckedHeader(extractor)
+					addFetchedLog(extractor)
+					addTransformerConfig(extractor)
+					mockLogRepository := &fakes.MockHeaderSyncLogRepository{}
+					mockLogRepository.CreateError = fakes.FakeError
+					extractor.LogRepository = mockLogRepository
+
+					err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+					Expect(err).To(HaveOccurred())
+					Expect(err).To(MatchError(fakes.FakeError))
+				})
+			})
+
+			It("marks header checked", func() {
+				addFetchedLog(extractor)
+				addTransformerConfig(extractor)
+				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+				headerID := rand.Int63()
+				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{Id: headerID}}
+				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).NotTo(HaveOccurred())
+				Expect(mockCheckedHeadersRepository.MarkHeaderCheckedHeaderID).To(Equal(headerID))
+			})
+
+			It("returns error if marking header checked fails", func() {
+				addFetchedLog(extractor)
+				addTransformerConfig(extractor)
+				mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+				mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{Id: rand.Int63()}}
+				mockCheckedHeadersRepository.MarkHeaderCheckedReturnError = fakes.FakeError
+				extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).To(HaveOccurred())
+				Expect(err).To(MatchError(fakes.FakeError))
+			})
+
+			It("returns nil for error if everything succeeds", func() {
+				addUncheckedHeader(extractor)
+				addTransformerConfig(extractor)
+
+				err := extractor.ExtractLogs(constants.HeaderUnchecked)
+
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+	})
+})
+
+func addTransformerConfig(extractor *logs.LogExtractor) {
+	fakeConfig := transformer.EventTransformerConfig{
+		ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+		Topic:               fakes.FakeHash.Hex(),
+		StartingBlockNumber: rand.Int63(),
+	}
+	extractor.AddTransformerConfig(fakeConfig)
+}
+
+func addUncheckedHeader(extractor *logs.LogExtractor) {
+	mockCheckedHeadersRepository := &fakes.MockCheckedHeadersRepository{}
+	mockCheckedHeadersRepository.UncheckedHeadersReturnHeaders = []core.Header{{}}
+	extractor.CheckedHeadersRepository = mockCheckedHeadersRepository
+}
+
+func addFetchedLog(extractor *logs.LogExtractor) {
+	mockLogFetcher := &mocks.MockLogFetcher{}
+	mockLogFetcher.ReturnLogs = []types.Log{{}}
+	extractor.Fetcher = mockLogFetcher
+}
+
+func getTransformerConfig(startingBlockNumber int64) transformer.EventTransformerConfig {
+	return transformer.EventTransformerConfig{
+		ContractAddresses:   []string{fakes.FakeAddress.Hex()},
+		Topic:               fakes.FakeHash.Hex(),
+		StartingBlockNumber: startingBlockNumber,
+	}
+}

libraries/shared/logs/logs_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package logs_test
+
+import (
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestLogs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Logs Suite")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

libraries/shared/mocks/converter.go

@@ -1,58 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-type MockConverter struct {
-	ToEntitiesError         error
-	PassedContractAddresses []string
-	ToModelsError           error
-	entityConverterError    error
-	modelConverterError     error
-	ContractAbi             string
-	LogsToConvert           []types.Log
-	EntitiesToConvert       []interface{}
-	EntitiesToReturn        []interface{}
-	ModelsToReturn          []interface{}
-	ToEntitiesCalledCounter int
-	ToModelsCalledCounter   int
-}
-
-func (converter *MockConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]interface{}, error) {
-	for _, log := range ethLogs {
-		converter.PassedContractAddresses = append(converter.PassedContractAddresses, log.Address.Hex())
-	}
-	converter.ContractAbi = contractAbi
-	converter.LogsToConvert = ethLogs
-	return converter.EntitiesToReturn, converter.ToEntitiesError
-}
-
-func (converter *MockConverter) ToModels(entities []interface{}) ([]interface{}, error) {
-	converter.EntitiesToConvert = entities
-	return converter.ModelsToReturn, converter.ToModelsError
-}
-
-func (converter *MockConverter) SetToEntityConverterError(err error) {
-	converter.entityConverterError = err
-}
-
-func (converter *MockConverter) SetToModelConverterError(err error) {
-	converter.modelConverterError = err
-}

libraries/shared/mocks/event_converter.go

@@ -0,0 +1,44 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockConverter struct {
+	ToModelsError           error
+	ContractAbi             string
+	LogsToConvert           []core.HeaderSyncLog
+	ModelsToReturn          []event.InsertionModel
+	PassedContractAddresses []string
+	SetDBCalled             bool
+	ToModelsCalledCounter   int
+}
+
+func (converter *MockConverter) ToModels(abi string, logs []core.HeaderSyncLog) ([]event.InsertionModel, error) {
+	converter.LogsToConvert = logs
+	converter.ContractAbi = abi
+	converter.ToModelsCalledCounter = converter.ToModelsCalledCounter + 1
+	return converter.ModelsToReturn, converter.ToModelsError
+}
+
+func (converter *MockConverter) SetDB(db *postgres.DB) {
+	converter.SetDBCalled = true
+}

libraries/shared/mocks/event_repository.go

@@ -0,0 +1,44 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockEventRepository struct {
+	createError         error
+	PassedModels        []event.InsertionModel
+	SetDbCalled         bool
+	CreateCalledCounter int
+}
+
+func (repository *MockEventRepository) Create(models []event.InsertionModel) error {
+	repository.PassedModels = models
+	repository.CreateCalledCounter++
+
+	return repository.createError
+}
+
+func (repository *MockEventRepository) SetDB(db *postgres.DB) {
+	repository.SetDbCalled = true
+}
+
+func (repository *MockEventRepository) SetCreateError(e error) {
+	repository.createError = e
+}

libraries/shared/mocks/event_transformer.go

@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+type MockEventTransformer struct {
+	ExecuteWasCalled bool
+	ExecuteError     error
+	PassedLogs       []core.HeaderSyncLog
+	config           transformer.EventTransformerConfig
+}
+
+func (t *MockEventTransformer) Execute(logs []core.HeaderSyncLog) error {
+	if t.ExecuteError != nil {
+		return t.ExecuteError
+	}
+	t.ExecuteWasCalled = true
+	t.PassedLogs = logs
+	return nil
+}
+
+func (t *MockEventTransformer) GetConfig() transformer.EventTransformerConfig {
+	return t.config
+}
+
+func (t *MockEventTransformer) SetTransformerConfig(config transformer.EventTransformerConfig) {
+	t.config = config
+}
+
+func (t *MockEventTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.EventTransformer {
+	return t
+}
+
+var FakeTransformerConfig = transformer.EventTransformerConfig{
+	TransformerName:   "FakeTransformer",
+	ContractAddresses: []string{fakes.FakeAddress.Hex()},
+	Topic:             fakes.FakeHash.Hex(),
+}

libraries/shared/mocks/log_delegator.go

@@ -0,0 +1,45 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+)
+
+type MockLogDelegator struct {
+	AddedTransformers []transformer.EventTransformer
+	DelegateCallCount int
+	DelegateErrors    []error
+}
+
+func (delegator *MockLogDelegator) AddTransformer(t transformer.EventTransformer) {
+	delegator.AddedTransformers = append(delegator.AddedTransformers, t)
+}
+
+func (delegator *MockLogDelegator) DelegateLogs() error {
+	delegator.DelegateCallCount++
+	if len(delegator.DelegateErrors) > 1 {
+		var delegateErrorThisRun error
+		delegateErrorThisRun, delegator.DelegateErrors = delegator.DelegateErrors[0], delegator.DelegateErrors[1:]
+		return delegateErrorThisRun
+	} else if len(delegator.DelegateErrors) == 1 {
+		thisErr := delegator.DelegateErrors[0]
+		delegator.DelegateErrors = []error{}
+		return thisErr
+	}
+	return nil
+}

libraries/shared/mocks/log_extractor.go

@@ -0,0 +1,48 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
+)
+
+type MockLogExtractor struct {
+	AddedConfigs              []transformer.EventTransformerConfig
+	AddTransformerConfigError error
+	ExtractLogsCount          int
+	ExtractLogsErrors         []error
+}
+
+func (extractor *MockLogExtractor) AddTransformerConfig(config transformer.EventTransformerConfig) error {
+	extractor.AddedConfigs = append(extractor.AddedConfigs, config)
+	return extractor.AddTransformerConfigError
+}
+
+func (extractor *MockLogExtractor) ExtractLogs(recheckHeaders constants.TransformerExecution) error {
+	extractor.ExtractLogsCount++
+	if len(extractor.ExtractLogsErrors) > 1 {
+		var errorThisRun error
+		errorThisRun, extractor.ExtractLogsErrors = extractor.ExtractLogsErrors[0], extractor.ExtractLogsErrors[1:]
+		return errorThisRun
+	} else if len(extractor.ExtractLogsErrors) == 1 {
+		thisErr := extractor.ExtractLogsErrors[0]
+		extractor.ExtractLogsErrors = []error{}
+		return thisErr
+	}
+	return nil
+}

libraries/shared/mocks/log_fetcher.go

@@ -17,26 +17,24 @@
 package mocks
 
 import (
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-type MockLogNoteConverter struct {
+type MockLogFetcher struct {
-	err                   error
+	ContractAddresses []common.Address
-	returnModels          []interface{}
+	FetchCalled       bool
-	PassedLogs            []types.Log
+	MissingHeader     core.Header
-	ToModelsCalledCounter int
+	ReturnError       error
+	ReturnLogs        []types.Log
+	Topics            []common.Hash
 }
 
-func (converter *MockLogNoteConverter) ToModels(ethLogs []types.Log) ([]interface{}, error) {
+func (fetcher *MockLogFetcher) FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error) {
-	converter.PassedLogs = ethLogs
+	fetcher.FetchCalled = true
-	converter.ToModelsCalledCounter++
+	fetcher.ContractAddresses = contractAddresses
-	return converter.returnModels, converter.err
+	fetcher.Topics = topics
-}
+	fetcher.MissingHeader = missingHeader
-
+	return fetcher.ReturnLogs, fetcher.ReturnError
-func (converter *MockLogNoteConverter) SetConverterError(e error) {
-	converter.err = e
-}
-
-func (converter *MockLogNoteConverter) SetReturnModels(models []interface{}) {
-	converter.returnModels = models
 }

libraries/shared/mocks/repository.go

@@ -1,98 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type MockRepository struct {
-	createError                      error
-	markHeaderCheckedError           error
-	MarkHeaderCheckedPassedHeaderIDs []int64
-	CreatedHeaderIds                 []int64
-	missingHeaders                   []core.Header
-	allHeaders                       []core.Header
-	missingHeadersError              error
-	PassedStartingBlockNumber        int64
-	PassedEndingBlockNumber          int64
-	PassedHeaderID                   int64
-	PassedModels                     []interface{}
-	SetDbCalled                      bool
-	CreateCalledCounter              int
-}
-
-func (repository *MockRepository) Create(headerID int64, models []interface{}) error {
-	repository.PassedHeaderID = headerID
-	repository.PassedModels = models
-	repository.CreatedHeaderIds = append(repository.CreatedHeaderIds, headerID)
-	repository.CreateCalledCounter++
-
-	return repository.createError
-}
-
-func (repository *MockRepository) MarkHeaderChecked(headerID int64) error {
-	repository.MarkHeaderCheckedPassedHeaderIDs = append(repository.MarkHeaderCheckedPassedHeaderIDs, headerID)
-	return repository.markHeaderCheckedError
-}
-
-func (repository *MockRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
-	repository.PassedStartingBlockNumber = startingBlockNumber
-	repository.PassedEndingBlockNumber = endingBlockNumber
-	return repository.missingHeaders, repository.missingHeadersError
-}
-
-func (repository *MockRepository) RecheckHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
-	repository.PassedStartingBlockNumber = startingBlockNumber
-	repository.PassedEndingBlockNumber = endingBlockNumber
-	return repository.allHeaders, nil
-}
-
-func (repository *MockRepository) SetDB(db *postgres.DB) {
-	repository.SetDbCalled = true
-}
-
-func (repository *MockRepository) SetMissingHeadersError(e error) {
-	repository.missingHeadersError = e
-}
-
-func (repository *MockRepository) SetAllHeaders(headers []core.Header) {
-	repository.allHeaders = headers
-}
-
-func (repository *MockRepository) SetMissingHeaders(headers []core.Header) {
-	repository.missingHeaders = headers
-}
-
-func (repository *MockRepository) SetMarkHeaderCheckedError(e error) {
-	repository.markHeaderCheckedError = e
-}
-
-func (repository *MockRepository) SetCreateError(e error) {
-	repository.createError = e
-}
-
-func (repository *MockRepository) AssertMarkHeaderCheckedCalledWith(i int64) {
-	Expect(repository.MarkHeaderCheckedPassedHeaderIDs).To(ContainElement(i))
-}
-
-func (repository *MockRepository) AssertMarkHeaderCheckedNotCalled() {
-	Expect(len(repository.MarkHeaderCheckedPassedHeaderIDs)).To(Equal(0))
-}

libraries/shared/mocks/storage_fetcher.go

@@ -19,21 +19,21 @@ package mocks
 import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 
 type MockStorageFetcher struct {
-	RowsToReturn []utils.StorageDiffRow
+	DiffsToReturn []utils.StorageDiff
-	ErrsToReturn []error
+	ErrsToReturn  []error
 }
 
 func NewMockStorageFetcher() *MockStorageFetcher {
 	return &MockStorageFetcher{}
 }
 
-func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	defer close(out)
 	defer close(errs)
 	for _, err := range fetcher.ErrsToReturn {
 		errs <- err
 	}
-	for _, row := range fetcher.RowsToReturn {
+	for _, diff := range fetcher.DiffsToReturn {
-		out <- row
+		out <- diff
 	}
 }

libraries/shared/mocks/storage_keys_loader.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+type MockStorageKeysLoader struct {
+	LoadMappingsCallCount int
+	LoadMappingsError     error
+	SetDBCalled           bool
+	StorageKeyMappings    map[common.Hash]utils.StorageValueMetadata
+}
+
+func (loader *MockStorageKeysLoader) LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error) {
+	loader.LoadMappingsCallCount++
+	return loader.StorageKeyMappings, loader.LoadMappingsError
+}
+
+func (loader *MockStorageKeysLoader) SetDB(db *postgres.DB) {
+	loader.SetDBCalled = true
+}

libraries/shared/mocks/storage_keys_lookup.go

@@ -18,22 +18,21 @@ package mocks
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-type MockMappings struct {
+type MockStorageKeysLookup struct {
 	Metadata     utils.StorageValueMetadata
 	LookupCalled bool
 	LookupErr    error
 }
 
-func (mappings *MockMappings) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
+func (mappings *MockStorageKeysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
 	mappings.LookupCalled = true
 	return mappings.Metadata, mappings.LookupErr
 }
 
-func (*MockMappings) SetDB(db *postgres.DB) {
+func (*MockStorageKeysLookup) SetDB(db *postgres.DB) {
 	panic("implement me")
 }

libraries/shared/mocks/storage_queue.go

@@ -23,16 +23,16 @@ import (
 type MockStorageQueue struct {
 	AddCalled      bool
 	AddError       error
-	AddPassedRow   utils.StorageDiffRow
+	AddPassedDiff  utils.StorageDiff
 	DeleteErr      error
 	DeletePassedId int
 	GetAllErr      error
-	RowsToReturn   []utils.StorageDiffRow
+	DiffsToReturn  []utils.StorageDiff
 }
 
-func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error {
+func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
 	queue.AddCalled = true
-	queue.AddPassedRow = row
+	queue.AddPassedDiff = diff
 	return queue.AddError
 }
 
@@ -41,6 +41,6 @@ func (queue *MockStorageQueue) Delete(id int) error {
 	return queue.DeleteErr
 }
 
-func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
+func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
-	return queue.RowsToReturn, queue.GetAllErr
+	return queue.DiffsToReturn, queue.GetAllErr
 }

libraries/shared/mocks/storage_transformer.go

@@ -18,25 +18,24 @@ package mocks
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type MockStorageTransformer struct {
-	Address    common.Address
+	KeccakOfAddress common.Hash
-	ExecuteErr error
+	ExecuteErr      error
-	PassedRow  utils.StorageDiffRow
+	PassedDiff      utils.StorageDiff
 }
 
-func (transformer *MockStorageTransformer) Execute(row utils.StorageDiffRow) error {
+func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
-	transformer.PassedRow = row
+	transformer.PassedDiff = diff
 	return transformer.ExecuteErr
 }
 
-func (transformer *MockStorageTransformer) ContractAddress() common.Address {
+func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash {
-	return transformer.Address
+	return transformer.KeccakOfAddress
 }
 
 func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {

libraries/shared/mocks/transformer.go

@@ -1,62 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type MockTransformer struct {
-	ExecuteWasCalled bool
-	ExecuteError     error
-	PassedLogs       []types.Log
-	PassedHeader     core.Header
-	config           shared_t.EventTransformerConfig
-}
-
-func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
-	if mh.ExecuteError != nil {
-		return mh.ExecuteError
-	}
-	mh.ExecuteWasCalled = true
-	mh.PassedLogs = logs
-	mh.PassedHeader = header
-	return nil
-}
-
-func (mh *MockTransformer) GetConfig() shared_t.EventTransformerConfig {
-	return mh.config
-}
-
-func (mh *MockTransformer) SetTransformerConfig(config shared_t.EventTransformerConfig) {
-	mh.config = config
-}
-
-func (mh *MockTransformer) FakeTransformerInitializer(db *postgres.DB) shared_t.EventTransformer {
-	return mh
-}
-
-var FakeTransformerConfig = shared_t.EventTransformerConfig{
-	TransformerName:   "FakeTransformer",
-	ContractAddresses: []string{"FakeAddress"},
-	Topic:             "FakeTopic",
-}

libraries/shared/mocks/watcher_repository.go

@@ -1,69 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package mocks
-
-import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type MockWatcherRepository struct {
-	ReturnCheckedColumnNames    []string
-	GetCheckedColumnNamesError  error
-	GetCheckedColumnNamesCalled bool
-
-	ReturnNotCheckedSQL       string
-	CreateNotCheckedSQLCalled bool
-
-	ReturnMissingHeaders []core.Header
-	MissingHeadersError  error
-	MissingHeadersCalled bool
-}
-
-func (repository *MockWatcherRepository) GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
-	repository.GetCheckedColumnNamesCalled = true
-	if repository.GetCheckedColumnNamesError != nil {
-		return []string{}, repository.GetCheckedColumnNamesError
-	}
-
-	return repository.ReturnCheckedColumnNames, nil
-}
-
-func (repository *MockWatcherRepository) SetCheckedColumnNames(checkedColumnNames []string) {
-	repository.ReturnCheckedColumnNames = checkedColumnNames
-}
-
-func (repository *MockWatcherRepository) CreateNotCheckedSQL(boolColumns []string) string {
-	repository.CreateNotCheckedSQLCalled = true
-	return repository.ReturnNotCheckedSQL
-}
-
-func (repository *MockWatcherRepository) SetNotCheckedSQL(notCheckedSql string) {
-	repository.ReturnNotCheckedSQL = notCheckedSql
-}
-
-func (repository *MockWatcherRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
-	if repository.MissingHeadersError != nil {
-		return []core.Header{}, repository.MissingHeadersError
-	}
-	repository.MissingHeadersCalled = true
-	return repository.ReturnMissingHeaders, nil
-}
-
-func (repository *MockWatcherRepository) SetMissingHeaders(headers []core.Header) {
-	repository.ReturnMissingHeaders = headers
-}

libraries/shared/repository/address_repository.go

@@ -0,0 +1,73 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/jmoiron/sqlx"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+const getOrCreateAddressQuery = `WITH addressId AS (
+			INSERT INTO addresses (address) VALUES ($1) ON CONFLICT DO NOTHING RETURNING id
+		)
+		SELECT id FROM addresses WHERE address = $1
+		UNION
+		SELECT id FROM addressId`
+
+func GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
+	checksumAddress := getChecksumAddress(address)
+
+	var addressId int64
+	getOrCreateErr := db.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
+
+	return addressId, getOrCreateErr
+}
+
+func GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
+	checksumAddress := getChecksumAddress(address)
+
+	var addressId int64
+	getOrCreateErr := tx.Get(&addressId, getOrCreateAddressQuery, checksumAddress)
+
+	return addressId, getOrCreateErr
+}
+
+func GetAddressById(db *postgres.DB, id int64) (string, error) {
+	var address string
+	getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
+	return address, getErr
+}
+
+func getChecksumAddress(address string) string {
+	stringAddressToCommonAddress := common.HexToAddress(address)
+	return stringAddressToCommonAddress.Hex()
+}

libraries/shared/repository/address_repository_test.go

@@ -0,0 +1,179 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package repository_test
+
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/test_config"
+)
+
+var _ = Describe("address lookup", func() {
+	var (
+		db      *postgres.DB
+		address = fakes.FakeAddress.Hex()
+	)
+	BeforeEach(func() {
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+	})
+
+	AfterEach(func() {
+		test_config.CleanTestDB(db)
+	})
+
+	type dbAddress struct {
+		Id      int64
+		Address string
+	}
+
+	Describe("GetOrCreateAddress", func() {
+		It("creates an address record", func() {
+			addressId, createErr := repository.GetOrCreateAddress(db, address)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			var actualAddress dbAddress
+			getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)
+			Expect(getErr).NotTo(HaveOccurred())
+			expectedAddress := dbAddress{Id: addressId, Address: address}
+			Expect(actualAddress).To(Equal(expectedAddress))
+		})
+
+		It("returns the existing record id if the address already exists", func() {
+			createId, createErr := repository.GetOrCreateAddress(db, address)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			getId, getErr := repository.GetOrCreateAddress(db, address)
+			Expect(getErr).NotTo(HaveOccurred())
+
+			var addressCount int
+			addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`)
+			Expect(addressErr).NotTo(HaveOccurred())
+			Expect(addressCount).To(Equal(1))
+			Expect(createId).To(Equal(getId))
+		})
+
+		It("gets upper-cased addresses", func() {
+			upperAddress := strings.ToUpper(address)
+			upperAddressId, createErr := repository.GetOrCreateAddress(db, upperAddress)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
+		})
+
+		It("gets lower-cased addresses", func() {
+			lowerAddress := strings.ToLower(address)
+			upperAddressId, createErr := repository.GetOrCreateAddress(db, lowerAddress)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddress(db, address)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
+		})
+	})
+
+	Describe("GetOrCreateAddressInTransaction", func() {
+		var (
+			tx    *sqlx.Tx
+			txErr error
+		)
+		BeforeEach(func() {
+			tx, txErr = db.Beginx()
+			Expect(txErr).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			tx.Rollback()
+		})
+
+		It("creates an address record", func() {
+			addressId, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
+			Expect(createErr).NotTo(HaveOccurred())
+			commitErr := tx.Commit()
+			Expect(commitErr).NotTo(HaveOccurred())
+
+			var actualAddress dbAddress
+			getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)
+			Expect(getErr).NotTo(HaveOccurred())
+			expectedAddress := dbAddress{Id: addressId, Address: address}
+			Expect(actualAddress).To(Equal(expectedAddress))
+		})
+
+		It("returns the existing record id if the address already exists", func() {
+			_, createErr := repository.GetOrCreateAddressInTransaction(tx, address)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			_, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
+			Expect(getErr).NotTo(HaveOccurred())
+			tx.Commit()
+
+			var addressCount int
+			addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`)
+			Expect(addressErr).NotTo(HaveOccurred())
+		})
+
+		It("gets upper-cased addresses", func() {
+			upperAddress := strings.ToUpper(address)
+			upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, upperAddress)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
+			Expect(getErr).NotTo(HaveOccurred())
+			tx.Commit()
+
+			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
+		})
+
+		It("gets lower-cased addresses", func() {
+			lowerAddress := strings.ToLower(address)
+			upperAddressId, createErr := repository.GetOrCreateAddressInTransaction(tx, lowerAddress)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			mixedCaseAddressId, getErr := repository.GetOrCreateAddressInTransaction(tx, address)
+			Expect(getErr).NotTo(HaveOccurred())
+			tx.Commit()
+
+			Expect(upperAddressId).To(Equal(mixedCaseAddressId))
+		})
+	})
+
+	Describe("GetAddressById", func() {
+		It("gets and address by it's id", func() {
+			addressId, createErr := repository.GetOrCreateAddress(db, address)
+			Expect(createErr).NotTo(HaveOccurred())
+
+			actualAddress, getErr := repository.GetAddressById(db, addressId)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(actualAddress).To(Equal(address))
+		})
+
+		It("returns an error if the id doesn't exist", func() {
+			_, getErr := repository.GetAddressById(db, 0)
+			Expect(getErr).To(HaveOccurred())
+			Expect(getErr).To(MatchError("sql: no rows in result set"))
+		})
+	})
+})

libraries/shared/repository/repository.go

@@ -1,157 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository
-
-import (
-	"bytes"
-	"database/sql/driver"
-	"fmt"
-	"github.com/jmoiron/sqlx"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
-	_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
-		VALUES ($1, $2)
-		ON CONFLICT (header_id) DO
-			UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
-	return err
-}
-
-func MarkHeaderCheckedInTransaction(headerID int64, tx *sqlx.Tx, checkedHeadersColumn string) error {
-	_, err := tx.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
-		VALUES ($1, $2)
-		ON CONFLICT (header_id) DO
-			UPDATE SET `+checkedHeadersColumn+` = checked_headers.`+checkedHeadersColumn+` + 1`, headerID, 1)
-	return err
-}
-
-// Treats a header as missing if it's not in the headers table, or not checked for some log type
-func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, notCheckedSQL string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-
-	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
-				AND headers.block_number >= $1
-				AND headers.eth_node_fingerprint = $2`
-		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
-	} else {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE (header_id ISNULL OR ` + notCheckedSQL + `)
-				AND headers.block_number >= $1
-				AND headers.block_number <= $2
-				AND headers.eth_node_fingerprint = $3`
-		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
-	}
-
-	return result, err
-}
-
-func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-
-	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
-				AND headers.block_number >= $1
-				AND headers.eth_node_fingerprint = $2`
-		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
-	} else {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
-				AND headers.block_number >= $1
-				AND headers.block_number <= $2
-				AND headers.eth_node_fingerprint = $3`
-		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
-	}
-
-	return result, err
-}
-
-func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
-	// Query returns `[]driver.Value`, nullable polymorphic interface
-	var queryResult []driver.Value
-	columnNamesQuery :=
-		`SELECT column_name FROM information_schema.columns
-		WHERE table_schema = 'public'
-			AND table_name = 'checked_headers'
-			AND column_name ~ '_checked';`
-
-	err := db.Select(&queryResult, columnNamesQuery)
-	if err != nil {
-		return []string{}, err
-	}
-
-	// Transform column names from `driver.Value` to strings
-	var columnNames []string
-	for _, result := range queryResult {
-		if columnName, ok := result.(string); ok {
-			columnNames = append(columnNames, columnName)
-		} else {
-			return []string{}, fmt.Errorf("incorrect value for checked_headers column name")
-		}
-	}
-
-	return columnNames, nil
-}
-
-// Builds a SQL string that checks if any column value is 0, given the column names.
-// Defaults to FALSE when no columns are provided.
-// Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
-//     [] => "FALSE"
-func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
-
-	var result bytes.Buffer
-
-	if len(boolColumns) == 0 {
-		return "FALSE"
-	}
-
-	result.WriteString("NOT (")
-
-	// Loop excluding last column name
-	for _, column := range boolColumns[:len(boolColumns)-1] {
-
-		if recheckHeaders {
-			result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
-		} else {
-			result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
-		}
-	}
-
-	// No trailing "OR" for the last column name
-	if recheckHeaders {
-		result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
-	} else {
-		result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
-
-	}
-
-	return result.String()
-}

libraries/shared/repository/repository_test.go

@@ -1,325 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package repository_test
-
-import (
-	"fmt"
-	"math/rand"
-	"strconv"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
-	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-	"github.com/vulcanize/vulcanizedb/test_config"
-)
-
-var _ = Describe("Repository", func() {
-	Describe("MarkHeaderChecked", func() {
-		var (
-			checkedHeadersColumn string
-			db                   *postgres.DB
-		)
-
-		BeforeEach(func() {
-			db = test_config.NewTestDB(test_config.NewTestNode())
-			test_config.CleanTestDB(db)
-
-			checkedHeadersColumn = "test_column_checked"
-			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
-				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
-			Expect(migrateErr).NotTo(HaveOccurred())
-		})
-
-		AfterEach(func() {
-			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
-			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
-		})
-
-		It("marks passed column as checked for passed header", func() {
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
-			Expect(headerErr).NotTo(HaveOccurred())
-
-			err := shared.MarkHeaderChecked(headerID, db, checkedHeadersColumn)
-
-			Expect(err).NotTo(HaveOccurred())
-			var checkedCount int
-			fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
-			Expect(fetchErr).NotTo(HaveOccurred())
-			Expect(checkedCount).To(Equal(1))
-		})
-	})
-
-	Describe("MarkHeaderCheckedInTransaction", func() {
-		var (
-			checkedHeadersColumn string
-			db                   *postgres.DB
-		)
-
-		BeforeEach(func() {
-			db = test_config.NewTestDB(test_config.NewTestNode())
-			test_config.CleanTestDB(db)
-
-			checkedHeadersColumn = "test_column_checked"
-			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers
-				ADD COLUMN ` + checkedHeadersColumn + ` integer`)
-			Expect(migrateErr).NotTo(HaveOccurred())
-		})
-
-		AfterEach(func() {
-			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
-			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
-		})
-
-		It("marks passed column as checked for passed header within a passed transaction", func() {
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerID, headerErr := headerRepository.CreateOrUpdateHeader(fakes.FakeHeader)
-			Expect(headerErr).NotTo(HaveOccurred())
-			tx, txErr := db.Beginx()
-			Expect(txErr).NotTo(HaveOccurred())
-
-			err := shared.MarkHeaderCheckedInTransaction(headerID, tx, checkedHeadersColumn)
-
-			Expect(err).NotTo(HaveOccurred())
-			commitErr := tx.Commit()
-			Expect(commitErr).NotTo(HaveOccurred())
-			var checkedCount int
-			fetchErr := db.Get(&checkedCount, `SELECT `+checkedHeadersColumn+` FROM public.checked_headers LIMIT 1`)
-			Expect(fetchErr).NotTo(HaveOccurred())
-			Expect(checkedCount).To(Equal(1))
-		})
-	})
-
-	Describe("MissingHeaders", func() {
-		var (
-			db                       *postgres.DB
-			headerRepository         datastore.HeaderRepository
-			startingBlockNumber      int64
-			endingBlockNumber        int64
-			eventSpecificBlockNumber int64
-			blockNumbers             []int64
-			headerIDs                []int64
-			notCheckedSQL            string
-			err                      error
-			hr                       r2.HeaderRepository
-		)
-
-		BeforeEach(func() {
-			db = test_config.NewTestDB(test_config.NewTestNode())
-			test_config.CleanTestDB(db)
-			headerRepository = repositories.NewHeaderRepository(db)
-			hr = r2.NewHeaderRepository(db)
-			hr.AddCheckColumns(getExpectedColumnNames())
-
-			columnNames, err := shared.GetCheckedColumnNames(db)
-			Expect(err).NotTo(HaveOccurred())
-			notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
-
-			startingBlockNumber = rand.Int63()
-			eventSpecificBlockNumber = startingBlockNumber + 1
-			endingBlockNumber = startingBlockNumber + 2
-			outOfRangeBlockNumber := endingBlockNumber + 1
-
-			blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
-
-			headerIDs = []int64{}
-			for _, n := range blockNumbers {
-				headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(n))
-				headerIDs = append(headerIDs, headerID)
-				Expect(err).NotTo(HaveOccurred())
-			}
-		})
-
-		AfterEach(func() {
-			test_config.CleanCheckedHeadersTable(db, getExpectedColumnNames())
-		})
-
-		It("only treats headers as checked if the event specific logs have been checked", func() {
-			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
-			Expect(err).NotTo(HaveOccurred())
-
-			headers, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(headers)).To(Equal(3))
-			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
-			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
-			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber)))
-		})
-
-		It("only returns headers associated with the current node", func() {
-			dbTwo := test_config.NewTestDB(core.Node{ID: "second"})
-			headerRepositoryTwo := repositories.NewHeaderRepository(dbTwo)
-			for _, n := range blockNumbers {
-				_, err = headerRepositoryTwo.CreateOrUpdateHeader(fakes.GetFakeHeader(n + 10))
-				Expect(err).NotTo(HaveOccurred())
-			}
-
-			Expect(err).NotTo(HaveOccurred())
-			nodeOneMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber, db, notCheckedSQL)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(nodeOneMissingHeaders)).To(Equal(3))
-			Expect(nodeOneMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
-			Expect(nodeOneMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
-			Expect(nodeOneMissingHeaders[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(startingBlockNumber), Equal(eventSpecificBlockNumber), Equal(endingBlockNumber)))
-
-			nodeTwoMissingHeaders, err := shared.MissingHeaders(startingBlockNumber, endingBlockNumber+10, dbTwo, notCheckedSQL)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(nodeTwoMissingHeaders)).To(Equal(3))
-			Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
-			Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
-		})
-	})
-
-	Describe("RecheckHeaders", func() {
-		var (
-			checkedHeadersColumn                                      string
-			db                                                        *postgres.DB
-			headerOneID, headerTwoID, headerThreeID, headerFourID     int64
-			headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
-		)
-
-		BeforeEach(func() {
-			db = test_config.NewTestDB(test_config.NewTestNode())
-			test_config.CleanTestDB(db)
-
-			// create header checked column
-			checkedHeadersColumn = "test_column_checked"
-			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
-			Expect(migrateErr).NotTo(HaveOccurred())
-
-			// create headers
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
-			Expect(headerOneErr).NotTo(HaveOccurred())
-			headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
-			Expect(headerTwoErr).NotTo(HaveOccurred())
-			headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
-			Expect(headerThreeErr).NotTo(HaveOccurred())
-			headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
-			Expect(headerFourErr).NotTo(HaveOccurred())
-
-			// mark every header checked at least once, with one fully rechecked (headerThree)
-			maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
-			Expect(intConversionErr).NotTo(HaveOccurred())
-			_, markHeaderOneCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerOneID, maxCheckCount)
-			Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderTwoCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerTwoID, maxCheckCount)
-			Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderThreeCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerThreeID, maxCheckCount+1)
-			Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderFourCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerFourID, maxCheckCount)
-			Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
-		})
-
-		AfterEach(func() {
-			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
-			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
-		})
-
-		Describe("when no ending block number (ending block number == -1)", func() {
-			It("returns all headers since starting block where checked count is less than cap", func() {
-				headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(headers)).To(Equal(3))
-				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
-				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
-				Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
-			})
-		})
-
-		Describe("when ending block number specified", func() {
-			It("returns headers between starting and ending block where checked count is less than cap", func() {
-				headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(headers)).To(Equal(2))
-				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
-				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
-			})
-		})
-	})
-
-	Describe("GetCheckedColumnNames", func() {
-		It("gets the column names from checked_headers", func() {
-			db := test_config.NewTestDB(test_config.NewTestNode())
-			hr := r2.NewHeaderRepository(db)
-			hr.AddCheckColumns(getExpectedColumnNames())
-			test_config.CleanTestDB(db)
-			expectedColumnNames := getExpectedColumnNames()
-			actualColumnNames, err := shared.GetCheckedColumnNames(db)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(actualColumnNames).To(Equal(expectedColumnNames))
-			test_config.CleanCheckedHeadersTable(db, getExpectedColumnNames())
-		})
-	})
-
-	Describe("CreateNotCheckedSQL", func() {
-		It("generates a correct SQL string for one column", func() {
-			columns := []string{"columnA"}
-			expected := "NOT (columnA!=0)"
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
-			Expect(actual).To(Equal(expected))
-		})
-
-		It("generates a correct SQL string for several columns", func() {
-			columns := []string{"columnA", "columnB"}
-			expected := "NOT (columnA!=0 AND columnB!=0)"
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
-			Expect(actual).To(Equal(expected))
-		})
-
-		It("defaults to FALSE when there are no columns", func() {
-			expected := "FALSE"
-			actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
-			Expect(actual).To(Equal(expected))
-		})
-
-		It("generates a correct SQL string for rechecking headers", func() {
-			columns := []string{"columnA", "columnB"}
-			expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
-			Expect(actual).To(Equal(expected))
-		})
-	})
-})
-
-func getExpectedColumnNames() []string {
-	return []string{
-		"column_1_checked",
-		"column_2_checked",
-		"column_3_checked",
-		"column_4_checked",
-	}
-}