Merge pull request #125 from vulcanize/decode-packed-addresses

(VDB-768) Allow addresses in packed slots to be decoded
2019-08-05 21:40:15 -05:00 · 2019-08-01 22:40:50 -05:00 · 2019-07-30 18:18:42 -07:00 · 2019-07-30 18:07:25 -07:00 · 2019-07-29 15:04:37 -05:00 · 2019-07-23 15:28:08 -05:00
5609 changed files with 2417 additions and 1843253 deletions
@@ -14,3 +14,5 @@ postgraphile/package-lock.json
 vulcanizedb.log
 db/migrations/20*.sql
 plugins/*.so
+postgraphile/*.toml
+postgraphile/schema.graphql
@@ -1,27 +1,24 @@
 dist: trusty
 language: go
 go:
-  - 1.11
+  - 1.12
 services:
  - postgresql
 addons:
-  postgresql: '9.6'
+  postgresql: '11.2'
 go_import_path: github.com/vulcanize/vulcanizedb
 before_install:
  - make installtools
-  - bash ./scripts/install-postgres-10.sh
+  - bash ./scripts/install-postgres-11.sh
  - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
  - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
  - sudo apt-get update && sudo apt-get install yarn
 before_script:
  - sudo -u postgres createdb vulcanize_private
  - make migrate NAME=vulcanize_private
-  - cd postgraphile && yarn
 script:
-  - yarn test
-  - cd ../
-  - make test
-  - make integrationtest
+  - env GO111MODULE=on make test
+  - env GO111MODULE=on make integrationtest
 notifications:
  email: false
 env:
@@ -1,564 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:48a213e9dc4880bbbd6999309a476fa4d3cc67560aa7127154cf8ea95bd464c2"
-  name = "github.com/allegro/bigcache"
-  packages = [
-    ".",
-    "queue",
-  ]
-  pruneopts = ""
-  revision = "f31987a23e44c5121ef8c8b2f2ea2e8ffa37b068"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
-  name = "github.com/aristanetworks/goarista"
-  packages = ["monotime"]
-  pruneopts = ""
-  revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
-  name = "github.com/btcsuite/btcd"
-  packages = ["btcec"]
-  pruneopts = ""
-  revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
-
-[[projects]]
-  digest = "1:5d47691333460db6ac83ced03c79b4bdb9aff3e322be24affb7855bed8affc6c"
-  name = "github.com/dave/jennifer"
-  packages = ["jen"]
-  pruneopts = ""
-  revision = "14e399b6b5e8456c66c45c955fc27b568bacb5c9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
-  name = "github.com/deckarep/golang-set"
-  packages = ["."]
-  pruneopts = ""
-  revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
-  version = "v1.7.1"
-
-[[projects]]
-  digest = "1:90d36f5b581e95e00ced808cd48824ed6c320c25887828cce461bdef4cb7bc7c"
-  name = "github.com/ethereum/go-ethereum"
-  packages = [
-    ".",
-    "accounts",
-    "accounts/abi",
-    "accounts/abi/bind",
-    "accounts/keystore",
-    "common",
-    "common/bitutil",
-    "common/hexutil",
-    "common/math",
-    "common/mclock",
-    "common/prque",
-    "core/rawdb",
-    "core/types",
-    "crypto",
-    "crypto/ecies",
-    "crypto/secp256k1",
-    "ethclient",
-    "ethdb",
-    "ethdb/leveldb",
-    "ethdb/memorydb",
-    "event",
-    "log",
-    "metrics",
-    "p2p",
-    "p2p/discover",
-    "p2p/discv5",
-    "p2p/enode",
-    "p2p/enr",
-    "p2p/nat",
-    "p2p/netutil",
-    "params",
-    "rlp",
-    "rpc",
-    "trie",
-  ]
-  pruneopts = ""
-  revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
-
-[[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  version = "v1.4.7"
-
-[[projects]]
-  digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
-  name = "github.com/go-stack/stack"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = ""
-  revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
-
-[[projects]]
-  digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = ""
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
-  name = "github.com/hashicorp/hcl"
-  packages = [
-    ".",
-    "hcl/ast",
-    "hcl/parser",
-    "hcl/printer",
-    "hcl/scanner",
-    "hcl/strconv",
-    "hcl/token",
-    "json/parser",
-    "json/scanner",
-    "json/token",
-  ]
-  pruneopts = ""
-  revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
-  name = "github.com/hpcloud/tail"
-  packages = [
-    ".",
-    "ratelimiter",
-    "util",
-    "watch",
-    "winfile",
-  ]
-  pruneopts = ""
-  revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
-  name = "github.com/huin/goupnp"
-  packages = [
-    ".",
-    "dcps/internetgateway1",
-    "dcps/internetgateway2",
-    "httpu",
-    "scpd",
-    "soap",
-    "ssdp",
-  ]
-  pruneopts = ""
-  revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
-
-[[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = ""
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
-  name = "github.com/jackpal/go-nat-pmp"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
-  name = "github.com/jmoiron/sqlx"
-  packages = [
-    ".",
-    "reflectx",
-  ]
-  pruneopts = ""
-  revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
-
-[[projects]]
-  digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3"
-  name = "github.com/konsorten/go-windows-terminal-sequences"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
-  name = "github.com/lib/pq"
-  packages = [
-    ".",
-    "oid",
-  ]
-  pruneopts = ""
-  revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
-
-[[projects]]
-  digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2353362d570a7bfa228149c62842019201cfb71"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  pruneopts = ""
-  revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
-  name = "github.com/onsi/ginkgo"
-  packages = [
-    ".",
-    "config",
-    "internal/codelocation",
-    "internal/containernode",
-    "internal/failer",
-    "internal/leafnodes",
-    "internal/remote",
-    "internal/spec",
-    "internal/spec_iterator",
-    "internal/specrunner",
-    "internal/suite",
-    "internal/testingtproxy",
-    "internal/writer",
-    "reporters",
-    "reporters/stenographer",
-    "reporters/stenographer/support/go-colorable",
-    "reporters/stenographer/support/go-isatty",
-    "types",
-  ]
-  pruneopts = ""
-  revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
-  version = "v1.6.0"
-
-[[projects]]
-  digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
-  name = "github.com/onsi/gomega"
-  packages = [
-    ".",
-    "format",
-    "ghttp",
-    "internal/assertion",
-    "internal/asyncassertion",
-    "internal/oraclematcher",
-    "internal/testingtsupport",
-    "matchers",
-    "matchers/support/goraph/bipartitegraph",
-    "matchers/support/goraph/edge",
-    "matchers/support/goraph/node",
-    "matchers/support/goraph/util",
-    "types",
-  ]
-  pruneopts = ""
-  revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
-  version = "v1.4.2"
-
-[[projects]]
-  digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
-  name = "github.com/pborman/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
-  version = "v1.2"
-
-[[projects]]
-  digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
-  name = "github.com/rjeczalik/notify"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
-  version = "v0.9.1"
-
-[[projects]]
-  digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
-  name = "github.com/rs/cors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
-  version = "v1.5.0"
-
-[[projects]]
-  digest = "1:9d57e200ef5ccc4217fe0a34287308bac652435e7c6513f6263e0493d2245c56"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = ""
-  revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
-  name = "github.com/spf13/afero"
-  packages = [
-    ".",
-    "mem",
-  ]
-  pruneopts = ""
-  revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
-  version = "v1.1.2"
-
-[[projects]]
-  digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8965335b8c7107321228e3e3702cab9832751bac"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  pruneopts = ""
-  revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = ""
-  revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
-  version = "v1.0.2"
-
-[[projects]]
-  digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
-  name = "github.com/syndtr/goleveldb"
-  packages = [
-    "leveldb",
-    "leveldb/cache",
-    "leveldb/comparer",
-    "leveldb/errors",
-    "leveldb/filter",
-    "leveldb/iterator",
-    "leveldb/journal",
-    "leveldb/memdb",
-    "leveldb/opt",
-    "leveldb/storage",
-    "leveldb/table",
-    "leveldb/util",
-  ]
-  pruneopts = ""
-  revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:59b49c47c11a48f1054529207f65907c014ecf5f9a7c0d9c0f1616dec7b062ed"
-  name = "golang.org/x/crypto"
-  packages = [
-    "pbkdf2",
-    "scrypt",
-    "sha3",
-    "ssh/terminal",
-  ]
-  pruneopts = ""
-  revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
-
-[[projects]]
-  branch = "master"
-  digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "html",
-    "html/atom",
-    "html/charset",
-    "websocket",
-  ]
-  pruneopts = ""
-  revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = ""
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows",
-  ]
-  pruneopts = ""
-  revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
-
-[[projects]]
-  digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
-  name = "golang.org/x/text"
-  packages = [
-    "encoding",
-    "encoding/charmap",
-    "encoding/htmlindex",
-    "encoding/internal",
-    "encoding/internal/identifier",
-    "encoding/japanese",
-    "encoding/korean",
-    "encoding/simplifiedchinese",
-    "encoding/traditionalchinese",
-    "encoding/unicode",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "internal/utf8internal",
-    "language",
-    "runes",
-    "transform",
-    "unicode/cldr",
-    "unicode/norm",
-  ]
-  pruneopts = ""
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
-  name = "gopkg.in/fsnotify.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  source = "gopkg.in/fsnotify/fsnotify.v1"
-  version = "v1.4.7"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
-  name = "gopkg.in/natefinch/npipe.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
-
-[[projects]]
-  branch = "v1"
-  digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
-  name = "gopkg.in/tomb.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
-
-[[projects]]
-  digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/dave/jennifer/jen",
-    "github.com/ethereum/go-ethereum",
-    "github.com/ethereum/go-ethereum/accounts/abi",
-    "github.com/ethereum/go-ethereum/accounts/abi/bind",
-    "github.com/ethereum/go-ethereum/common",
-    "github.com/ethereum/go-ethereum/common/hexutil",
-    "github.com/ethereum/go-ethereum/core/rawdb",
-    "github.com/ethereum/go-ethereum/core/types",
-    "github.com/ethereum/go-ethereum/crypto",
-    "github.com/ethereum/go-ethereum/ethclient",
-    "github.com/ethereum/go-ethereum/ethdb",
-    "github.com/ethereum/go-ethereum/p2p",
-    "github.com/ethereum/go-ethereum/p2p/discv5",
-    "github.com/ethereum/go-ethereum/params",
-    "github.com/ethereum/go-ethereum/rlp",
-    "github.com/ethereum/go-ethereum/rpc",
-    "github.com/hashicorp/golang-lru",
-    "github.com/hpcloud/tail",
-    "github.com/jmoiron/sqlx",
-    "github.com/lib/pq",
-    "github.com/mitchellh/go-homedir",
-    "github.com/onsi/ginkgo",
-    "github.com/onsi/gomega",
-    "github.com/onsi/gomega/ghttp",
-    "github.com/sirupsen/logrus",
-    "github.com/spf13/cobra",
-    "github.com/spf13/viper",
-    "golang.org/x/net/context",
-    "golang.org/x/sync/errgroup",
-    "gopkg.in/tomb.v1",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1
@@ -1,54 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[override]]
-  name = "gopkg.in/fsnotify.v1"
-  source = "gopkg.in/fsnotify/fsnotify.v1"
-
-[[override]]
-  name = "github.com/pressly/sup"
-  version = "0.5.3"
-
-[[constraint]]
-  name = "github.com/onsi/ginkgo"
-  version = "1.4.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jmoiron/sqlx"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/lib/pq"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "1.2.0"
-
-[[constraint]]
-  name = "github.com/spf13/cobra"
-  version = "0.0.1"
-
-[[constraint]]
-  name = "github.com/ethereum/go-ethereum"
-  revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
@@ -3,11 +3,6 @@ BASE = $(GOPATH)/src/$(PACKAGE)
 PKGS = go list ./... | grep -v "^vendor/"

 # Tools
-## Dependency management
-DEP = $(BIN)/dep
-$(BIN)/dep:
-	go get -u github.com/golang/dep/cmd/dep
-
 ## Testing library
 GINKGO = $(BIN)/ginkgo
 $(BIN)/ginkgo:
@@ -32,7 +27,7 @@ $(BIN)/gometalinter.v2:


 .PHONY: installtools
-installtools: | $(LINT) $(GOOSE) $(GINKGO) $(DEP)
+installtools: | $(LINT) $(GOOSE) $(GINKGO)
 	echo "Installing tools"

 .PHONY: metalint
@@ -58,11 +53,7 @@ integrationtest: | $(GINKGO) $(LINT)
 	go fmt ./...
 	$(GINKGO) -r integration_test/

-.PHONY: dep
-dep: | $(DEP)
-	$(DEP) ensure
-
-build: dep
+build:
 	go fmt ./...
 	go build

@@ -70,7 +61,8 @@ build: dep
 HOST_NAME = localhost
 PORT = 5432
 NAME =
-CONNECT_STRING=postgresql://$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+USER = postgres
+CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable

 # Parameter checks
 ## Check that DB variables are provided
@@ -79,6 +71,7 @@ checkdbvars:
 	test -n "$(HOST_NAME)" # $$HOST_NAME
 	test -n "$(PORT)" # $$PORT
 	test -n "$(NAME)" # $$NAME
+	test -n "$(USER)" # $$USER
 	@echo $(CONNECT_STRING)

 ## Check that the migration variable (id/timestamp) is provided
@@ -7,49 +7,56 @@


 ## Table of Contents
-1. [Background](../staging/README.md#background)
-1. [Dependencies](../staging/README.md#dependencies)
-1. [Install](../staging/README.md#install)
-1. [Usage](../staging/README.md#usage)
-1. [Tests](../staging/README.md#tests)
-1. [API](../staging/README.md#API)
-1. [Contributing](../staging/README.md#contributing)
-1. [License](../staging/README.md#license)
+1. [Background](#background)
+1. [Install](#install)
+1. [Usage](#usage)
+1. [Contributing](#contributing)
+1. [License](#license)


 ## Background
 The same data structures and encodings that make Ethereum an effective and trust-less distributed virtual machine
-complicate data accessibility and usability for dApp developers.
+complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by
+providing a suite of tools to ease the extraction and transformation of data into a more useful state, including
+allowing for exposing aggregate data from a suite of smart contracts.

+VulanizeDB includes processes that sync, transform and expose data. Syncing involves
+querying an Ethereum node and then persisting core data into a Postgres database. Transforming focuses on using previously synced data to
+query for and transform log event and storage data for specifically configured smart contract addresses. Exposing data is a matter of getting
+data from VulcanizeDB's underlying Postgres database and making it accessible.

-## Dependencies
- - Go 1.11+
- - Postgres 10.6
+![VulcanizeDB Overview Diagram](documentation/diagrams/vdb-overview.png)
+
+## Install
+
+1. [Dependencies](#dependencies)
+1. [Building the project](#building-the-project)
+1. [Setting up the database](#setting-up-the-database)
+1. [Configuring a synced Ethereum node](#configuring-a-synced-ethereum-node)
+
+### Dependencies
+ - Go 1.12+
+ - Postgres 11.2
 - Ethereum Node
   - [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.23+)
   - [Parity 1.8.11+](https://github.com/paritytech/parity/releases)

-
-## Install
-1. [Building the project](../staging/README.md#building-the-project)
-1. [Setting up the database](../staging/README.md#setting-up-the-database)
-1. [Configuring a synced Ethereum node](../staging/README.md#configuring-a-synced-ethereum-node)
-
 ### Building the project
 Download the codebase to your local `GOPATH` via:

 `go get github.com/vulcanize/vulcanizedb`

-Move to the project directory and use [golang/dep](https://github.com/golang/dep) to install the dependencies:
+Move to the project directory:

 `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`

-`dep ensure`
-
-Once the dependencies have been successfully installed, build the executable with:
+Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:

 `make build`

+If you need to use a different dependency than what is currently defined in `go.mod`, it may helpful to look into [the replace directive](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
+This instruction enables you to point at a fork or the local filesystem for dependency resolution.
+
 If you are running into issues at this stage, ensure that `GOPATH` is defined in your shell.
 If necessary, `GOPATH` can be set in `~/.bashrc` or `~/.bash_profile`, depending upon your system.
 It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
@@ -60,13 +67,15 @@ It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
 1. `createdb vulcanize_public`
 1. `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
 1.  Run the migrations: `make migrate HOST_NAME=localhost NAME=vulcanize_public PORT=5432`
+    - There is an optional var `USER=username` if the database user is not the default user `postgres`
    - To rollback a single step: `make rollback NAME=vulcanize_public`
    - To rollback to a certain migration: `make rollback_to MIGRATION=n NAME=vulcanize_public`
    - To see status of migrations: `make migration_status NAME=vulcanize_public`

    * See below for configuring additional environments
    
-In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
+In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from
+localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf

 (It should be noted that trusted auth should only be enabled on systems without sensitive data in them: development and local test databases)

@@ -78,8 +87,9 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
        - The IPC file is called `geth.ipc`.
        - The geth IPC file path is printed to the console when you start geth.
        - The default location is:
-          - Mac: `<full home path>/Library/Ethereum`
+          - Mac: `<full home path>/Library/Ethereum/geth.ipc`
          - Linux: `<full home path>/ethereum/geth.ipc`
+        - Note: the geth.ipc file may not exist until you've started the geth process

      - For Parity:
        - The IPC file is called `jsonrpc.ipc`.
@@ -96,23 +106,35 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f


 ## Usage
-Usage is broken up into two processes:
+As mentioned above, VulcanizeDB's processes can be split into three categories: syncing, transforming and exposing data.

 ### Data syncing
-To provide data for transformations, raw Ethereum data must first be synced into vDB.
-This is accomplished through the use of the `lightSync`, `sync`, or `coldImport` commands.
-These commands are described in detail [here](../staging/documentation/sync.md).
+To provide data for transformations, raw Ethereum data must first be synced into VulcanizeDB.
+This is accomplished through the use of the `headerSync`, `fullSync`, or `coldImport` commands.
+These commands are described in detail [here](documentation/data-syncing.md).

 ### Data transformation
-Contract watchers use the raw data that has been synced into Postgres to filter out and apply transformations to specific data of interest.
+Data transformation uses the raw data that has been synced into Postgres to filter out and apply transformations to
+specific data of interest. Since there are different types of data that may be useful for observing smart contracts, it
+follows that there are different ways to transform this data. We've started by categorizing this into Generic and
+Custom transformers:

-There is a built-in `contractWatcher` command which provides generic transformation of most contract data. This command is described in detail [here](../staging/documentation/contractWatcher.md).
+- Generic Contract Transformer: Generic contract transformation can be done using a built-in command,
+`contractWatcher`, which transforms contract events provided the contract's ABI is available. It also
+provides some state variable coverage by automating polling of public methods, with some restrictions.
+`contractWatcher` is described further [here](documentation/generic-transformer.md).

-In many cases a custom transformer or set of transformers will need to be written to provide complete or more comprehensive coverage or to optimize other aspects of the output for a specific end-use.
-In this case we have provided the `compose`, `execute`, and `composeAndExecute` commands for running custom transformers from external repositories. This is described in detail [here](../staging/documentation/composeAndExecute.md).
+- Custom Transformers: In many cases custom transformers will need to be written to provide
+more comprehensive coverage of contract data. In this case we have provided the `compose`, `execute`, and
+`composeAndExecute` commands for running custom transformers from external repositories. Documentation on how to write,
+build and run custom transformers as Go plugins can be found
+[here](documentation/custom-transformers.md).
+
+### Exposing the data
+[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](documentation/postgraphile.md).


-## Tests
+### Tests
 - Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url)
    - Note: integration tests require configuration with an archival node
 - `createdb vulcanize_private` will create the test db
@@ -120,15 +142,13 @@ In this case we have provided the `compose`, `execute`, and `composeAndExecute`
 - `make test` will run the unit tests and skip the integration tests
 - `make integrationtest` will run just the integration tests

-## API
-[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](../staging/postgraphile/README.md).
-

 ## Contributing
-Contributions are welcome! For more on this, please see [here](../staging/documentation/contributing.md).
+Contributions are welcome!

-Small note: If editing the Readme, please conform to the [standard-readme specification](https://github.com/RichardLitt/standard-readme).
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).

+For more information on contributing, please see [here](documentation/contributing.md).

 ## License
-[AGPL-3.0](../staging/LICENSE) © Vulcanize Inc
+[AGPL-3.0](LICENSE) © Vulcanize Inc
@@ -39,6 +39,8 @@ var coldImportCmd = &cobra.Command{

 Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		coldImport()
 	},
 }
@@ -55,7 +57,7 @@ func coldImport() {
 	ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
 	ethDB, err := ethereum.CreateDatabase(ethDBConfig)
 	if err != nil {
-		log.Fatal("Error connecting to ethereum db: ", err)
+		LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
 	}
 	mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
 	if syncAll {
@@ -63,10 +65,10 @@ func coldImport() {
 		endingBlockNumber = mostRecentBlockNumberInDb
 	}
 	if endingBlockNumber < startingBlockNumber {
-		log.Fatal("Ending block number must be greater than starting block number for cold import.")
+		LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
 	}
 	if endingBlockNumber > mostRecentBlockNumberInDb {
-		log.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
+		LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
 	}

 	// init pg db
@@ -76,7 +78,7 @@ func coldImport() {
 	nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
 	coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
 	if err != nil {
-		log.Fatal("Error getting node: ", err)
+		LogWithCommand.Fatal("Error getting node: ", err)
 	}
 	pgDB := utils.LoadPostgres(databaseConfig, coldNode)

@@ -90,6 +92,6 @@ func coldImport() {
 	coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
 	err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
 	if err != nil {
-		log.Fatal("Error executing cold import: ", err)
+		LogWithCommand.Fatal("Error executing cold import: ", err)
 	}
 }
@@ -101,6 +101,8 @@ single config file or in separate command instances using different config files
 Specify config location when executing the command:
 ./vulcanizedb compose --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		compose()
 	},
 }
@@ -110,23 +112,25 @@ func compose() {
 	prepConfig()

 	// Generate code to build the plugin according to the config file
-	log.Info("generating plugin")
+	LogWithCommand.Info("generating plugin")
 	generator, err := p2.NewGenerator(genConfig, databaseConfig)
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Debug("initializing plugin generator failed")
+		LogWithCommand.Fatal(err)
 	}
 	err = generator.GenerateExporterPlugin()
 	if err != nil {
-		log.Debug("generating plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Debug("generating plugin failed")
+		LogWithCommand.Fatal(err)
 	}
 	// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Debug("getting plugin path failed")
+		LogWithCommand.Fatal(err)
 	}
 	fmt.Printf("Composed plugin %s", pluginPath)
-	log.Info("plugin .so file output to", pluginPath)
+	LogWithCommand.Info("plugin .so file output to ", pluginPath)
 }

 func init() {
@@ -134,38 +138,38 @@ func init() {
 }

 func prepConfig() {
-	log.Info("configuring plugin")
+	LogWithCommand.Info("configuring plugin")
 	names := viper.GetStringSlice("exporter.transformerNames")
 	transformers := make(map[string]config.Transformer)
 	for _, name := range names {
 		transformer := viper.GetStringMapString("exporter." + name)
 		p, pOK := transformer["path"]
 		if !pOK || p == "" {
-			log.Fatal(name, "transformer config is missing `path` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `path` value")
 		}
 		r, rOK := transformer["repository"]
 		if !rOK || r == "" {
-			log.Fatal(name, "transformer config is missing `repository` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
 		}
 		m, mOK := transformer["migrations"]
 		if !mOK || m == "" {
-			log.Fatal(name, "transformer config is missing `migrations` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
 		}
 		mr, mrOK := transformer["rank"]
 		if !mrOK || mr == "" {
-			log.Fatal(name, "transformer config is missing `rank` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
 		}
 		rank, err := strconv.ParseUint(mr, 10, 64)
 		if err != nil {
-			log.Fatal(name, "migration `rank` can't be converted to an unsigned integer")
+			LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
 		}
 		t, tOK := transformer["type"]
 		if !tOK {
-			log.Fatal(name, "transformer config is missing `type` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `type` value")
 		}
 		transformerType := config.GetTransformerType(t)
 		if transformerType == config.UnknownTransformerType {
-			log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
+			LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
 		}

 		transformers[name] = config.Transformer{
@@ -16,16 +16,20 @@
 package cmd

 import (
+	"os"
+	"plugin"
+	syn "sync"
+	"time"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
 	p2 "github.com/vulcanize/vulcanizedb/pkg/plugin"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
 	"github.com/vulcanize/vulcanizedb/utils"
-	"os"
-	"plugin"
-	syn "sync"
 )

 // composeAndExecuteCmd represents the composeAndExecute command
@@ -101,6 +105,8 @@ single config file or in separate command instances using different config files
 Specify config location when executing the command:
 ./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		composeAndExecute()
 	},
 }
@@ -110,44 +116,44 @@ func composeAndExecute() {
 	prepConfig()

 	// Generate code to build the plugin according to the config file
-	log.Info("generating plugin")
+	LogWithCommand.Info("generating plugin")
 	generator, err := p2.NewGenerator(genConfig, databaseConfig)
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	err = generator.GenerateExporterPlugin()
 	if err != nil {
-		log.Debug("generating plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Debug("generating plugin failed")
+		LogWithCommand.Fatal(err)
 	}

 	// Get the plugin path and load the plugin
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	if !genConfig.Save {
 		defer helpers.ClearFiles(pluginPath)
 	}
-	log.Info("linking plugin", pluginPath)
+	LogWithCommand.Info("linking plugin ", pluginPath)
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
-		log.Debug("linking plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Debug("linking plugin failed")
+		LogWithCommand.Fatal(err)
 	}

 	// Load the `Exporter` symbol from the plugin
-	log.Info("loading transformers from plugin")
+	LogWithCommand.Info("loading transformers from plugin")
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
-		log.Debug("loading Exporter symbol failed")
-		log.Fatal(err)
+		LogWithCommand.Debug("loading Exporter symbol failed")
+		LogWithCommand.Fatal(err)
 	}

 	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
-		log.Debug("plugged-in symbol not of type Exporter")
+		LogWithCommand.Debug("plugged-in symbol not of type Exporter")
 		os.Exit(1)
 	}

@@ -170,7 +176,8 @@ func composeAndExecute() {

 	if len(ethStorageInitializers) > 0 {
 		tailer := fs.FileTailer{Path: storageDiffsPath}
-		sw := watcher.NewStorageWatcher(tailer, &db)
+		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+		sw := watcher.NewStorageWatcher(storageFetcher, &db)
 		sw.AddTransformers(ethStorageInitializers)
 		wg.Add(1)
 		go watchEthStorage(&sw, &wg)
@@ -187,5 +194,6 @@ func composeAndExecute() {

 func init() {
 	rootCmd.AddCommand(composeAndExecuteCmd)
-	composeAndExecuteCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
+	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
+	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
 }
@@ -26,7 +26,7 @@ import (

 	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
-	lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
+	ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/utils"
 )

@@ -79,6 +79,8 @@ Requires a .toml config file:
        piping = true
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		contractWatcher()
 	},
 }
@@ -98,28 +100,28 @@ func contractWatcher() {
 	con := config.ContractConfig{}
 	con.PrepConfig()
 	switch mode {
-	case "light":
-		t = lt.NewTransformer(con, blockChain, &db)
+	case "header":
+		t = ht.NewTransformer(con, blockChain, &db)
 	case "full":
 		t = ft.NewTransformer(con, blockChain, &db)
 	default:
-		log.Fatal("Invalid mode")
+		LogWithCommand.Fatal("Invalid mode")
 	}

 	err := t.Init()
 	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
+		LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
 	}

 	for range ticker.C {
 		err = t.Execute()
 		if err != nil {
-			log.Error("Execution error for transformer:", t.GetConfig().Name, err)
+			LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
 		}
 	}
 }

 func init() {
 	rootCmd.AddCommand(contractWatcherCmd)
-	contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "light", "'light' or 'full' mode to work with either light synced or fully synced vDB (default is light)")
+	contractWatcherCmd.Flags().StringVarP(&mode, "mode", "o", "header", "'header' or 'full' mode to work with either header synced or fully synced vDB (default is header)")
 }
@@ -17,7 +17,6 @@ package cmd

 import (
 	"fmt"
-	"os"
 	"plugin"
 	syn "sync"
 	"time"
@@ -26,6 +25,8 @@ import (
 	"github.com/spf13/cobra"

 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	storageUtils "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
@@ -62,6 +63,8 @@ must have been composed by the same version of vulcanizedb or else it will not b
 Specify config location when executing the command:
 ./vulcanizedb execute --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		execute()
 	},
 }
@@ -73,30 +76,29 @@ func execute() {
 	// Get the plugin path and load the plugin
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}

 	fmt.Printf("Executing plugin %s", pluginPath)
-	log.Info("linking plugin", pluginPath)
+	LogWithCommand.Info("linking plugin ", pluginPath)
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
-		log.Debug("linking plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Warn("linking plugin failed")
+		LogWithCommand.Fatal(err)
 	}

 	// Load the `Exporter` symbol from the plugin
-	log.Info("loading transformers from plugin")
+	LogWithCommand.Info("loading transformers from plugin")
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
-		log.Debug("loading Exporter symbol failed")
-		log.Fatal(err)
+		LogWithCommand.Warn("loading Exporter symbol failed")
+		LogWithCommand.Fatal(err)
 	}

 	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
-		log.Debug("plugged-in symbol not of type Exporter")
-		os.Exit(1)
+		LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
 	}

 	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
@@ -118,7 +120,8 @@ func execute() {

 	if len(ethStorageInitializers) > 0 {
 		tailer := fs.FileTailer{Path: storageDiffsPath}
-		sw := watcher.NewStorageWatcher(tailer, &db)
+		storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
+		sw := watcher.NewStorageWatcher(storageFetcher, &db)
 		sw.AddTransformers(ethStorageInitializers)
 		wg.Add(1)
 		go watchEthStorage(&sw, &wg)
@@ -135,7 +138,8 @@ func execute() {

 func init() {
 	rootCmd.AddCommand(executeCmd)
-	executeCmd.Flags().BoolVar(&recheckHeadersArg, "recheckHeaders", false, "checks headers that are already checked for each transformer.")
+	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
+	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
 }

 type Exporter interface {
@@ -145,7 +149,7 @@ type Exporter interface {
 func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the EventTransformerInitializer set using the watcher
-	log.Info("executing event transformers")
+	LogWithCommand.Info("executing event transformers")
 	var recheck constants.TransformerExecution
 	if recheckHeadersArg {
 		recheck = constants.HeaderRecheck
@@ -162,18 +166,20 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the StorageTransformerInitializer set using the storage watcher
-	log.Info("executing storage transformers")
+	LogWithCommand.Info("executing storage transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {
-		w.Execute()
+		errs := make(chan error)
+		rows := make(chan storageUtils.StorageDiffRow)
+		w.Execute(rows, errs, queueRecheckInterval)
 	}
 }

 func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the ContractTransformerInitializer set using the contract watcher
-	log.Info("executing contract_watcher transformers")
+	LogWithCommand.Info("executing contract_watcher transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {
@@ -29,14 +29,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/utils"
 )

-// syncCmd represents the sync command
-var syncCmd = &cobra.Command{
-	Use:   "sync",
+// fullSyncCmd represents the fullSync command
+var fullSyncCmd = &cobra.Command{
+	Use:   "fullSync",
 	Short: "Syncs VulcanizeDB with local ethereum node",
 	Long: `Syncs VulcanizeDB with local ethereum node. Populates
 Postgres with blocks, transactions, receipts, and logs.

-./vulcanizedb sync --starting-block-number 0 --config public.toml
+./vulcanizedb fullSync --starting-block-number 0 --config public.toml

 Expects ethereum node to be running and requires a .toml config:

@@ -49,38 +49,39 @@ Expects ethereum node to be running and requires a .toml config:
  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		sync()
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
+		fullSync()
 	},
 }

 func init() {
-	rootCmd.AddCommand(syncCmd)
-
-	syncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
+	rootCmd.AddCommand(fullSyncCmd)
+	fullSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
 }

 func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
 	populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
 	if err != nil {
-		log.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
+		LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
 	}
 	missingBlocksPopulated <- populated
 }

-func sync() {
+func fullSync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()

 	blockChain := getBlockChain()
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
-		log.Error("sync: Error getting last block: ", err)
+		LogWithCommand.Error("fullSync: Error getting last block: ", err)
 	}
 	if lastBlock.Int64() == 0 {
-		log.Fatal("geth initial: state sync not finished")
+		LogWithCommand.Fatal("geth initial: state sync not finished")
 	}
 	if startingBlockNumber > lastBlock.Int64() {
-		log.Fatal("sync: starting block number > current block number")
+		LogWithCommand.Fatal("fullSync: starting block number > current block number")
 	}

 	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
@@ -94,9 +95,9 @@ func sync() {
 		case <-ticker.C:
 			window, err := validator.ValidateBlocks()
 			if err != nil {
-				log.Error("sync: error in validateBlocks: ", err)
+				LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
 			}
-			log.Info(window.GetString())
+			LogWithCommand.Debug(window.GetString())
 		case <-missingBlocksPopulated:
 			go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
 		}
@@ -30,14 +30,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/utils"
 )

-// lightSyncCmd represents the lightSync command
-var lightSyncCmd = &cobra.Command{
-	Use:   "lightSync",
+// headerSyncCmd represents the headerSync command
+var headerSyncCmd = &cobra.Command{
+	Use:   "headerSync",
 	Short: "Syncs VulcanizeDB with local ethereum node's block headers",
 	Long: `Syncs VulcanizeDB with local ethereum node. Populates
 Postgres with block headers.

-./vulcanizedb lightSync --starting-block-number 0 --config public.toml
+./vulcanizedb headerSync --starting-block-number 0 --config public.toml

 Expects ethereum node to be running and requires a .toml config:

@@ -50,13 +50,15 @@ Expects ethereum node to be running and requires a .toml config:
  ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		lightSync()
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
+		headerSync()
 	},
 }

 func init() {
-	rootCmd.AddCommand(lightSyncCmd)
-	lightSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
+	rootCmd.AddCommand(headerSyncCmd)
+	headerSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
 }

 func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
@@ -64,12 +66,12 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
 	if err != nil {
 		// TODO Lots of possible errors in the call stack above. If errors occur, we still put
 		// 0 in the channel, triggering another round
-		log.Error("backfillAllHeaders: Error populating headers: ", err)
+		LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
 	}
 	missingBlocksPopulated <- populated
 }

-func lightSync() {
+func headerSync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	blockChain := getBlockChain()
@@ -86,9 +88,9 @@ func lightSync() {
 		case <-ticker.C:
 			window, err := validator.ValidateHeaders()
 			if err != nil {
-				log.Error("lightSync: ValidateHeaders failed: ", err)
+				LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
 			}
-			log.Info(window.GetString())
+			LogWithCommand.Debug(window.GetString())
 		case n := <-missingBlocksPopulated:
 			if n == 0 {
 				time.Sleep(3 * time.Second)
@@ -101,12 +103,12 @@ func lightSync() {
 func validateArgs(blockChain *geth.BlockChain) {
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
-		log.Error("validateArgs: Error getting last block: ", err)
+		LogWithCommand.Error("validateArgs: Error getting last block: ", err)
 	}
 	if lastBlock.Int64() == 0 {
-		log.Fatal("geth initial: state sync not finished")
+		LogWithCommand.Fatal("geth initial: state sync not finished")
 	}
 	if startingBlockNumber > lastBlock.Int64() {
-		log.Fatal("starting block number > current block number")
+		LogWithCommand.Fatal("starting block number > current block number")
 	}
 }
@@ -18,7 +18,6 @@ package cmd

 import (
 	"fmt"
-	"os"
 	"strings"
 	"time"

@@ -36,16 +35,19 @@ import (
 )

 var (
-	cfgFile             string
-	databaseConfig      config.Database
-	genConfig           config.Plugin
-	ipc                 string
-	levelDbPath         string
-	startingBlockNumber int64
-	storageDiffsPath    string
-	syncAll             bool
-	endingBlockNumber   int64
-	recheckHeadersArg   bool
+	cfgFile              string
+	databaseConfig       config.Database
+	genConfig            config.Plugin
+	ipc                  string
+	levelDbPath          string
+	queueRecheckInterval time.Duration
+	startingBlockNumber  int64
+	storageDiffsPath     string
+	syncAll              bool
+	endingBlockNumber    int64
+	recheckHeadersArg    bool
+	SubCommand           string
+	LogWithCommand       log.Entry
 )

 const (
@@ -55,18 +57,26 @@ const (

 var rootCmd = &cobra.Command{
 	Use:              "vulcanizedb",
-	PersistentPreRun: database,
+	PersistentPreRun: initFuncs,
 }

 func Execute() {
 	log.Info("----- Starting vDB -----")
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
-		os.Exit(1)
 	}
 }

-func database(cmd *cobra.Command, args []string) {
+func initFuncs(cmd *cobra.Command, args []string) {
+	setViperConfigs()
+	logLvlErr := logLevel()
+	if logLvlErr != nil {
+		log.Fatal("Could not set log level: ", logLvlErr)
+	}
+
+}
+
+func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
 	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
@@ -80,6 +90,19 @@ func database(cmd *cobra.Command, args []string) {
 	viper.Set("database.config", databaseConfig)
 }

+func logLevel() error {
+	lvl, err := log.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	log.SetLevel(lvl)
+	if lvl > log.InfoLevel {
+		log.SetReportCaller(true)
+	}
+	log.Info("Log level set to ", lvl.String())
+	return nil
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 	// When searching for env variables, replace dots in config keys with underscores
@@ -96,6 +119,7 @@ func init() {
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")

 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -106,6 +130,7 @@ func init() {
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }

 func initConfig() {
@@ -115,16 +140,14 @@ func initConfig() {
 		noConfigError := "No config file passed with --config flag"
 		fmt.Println("Error: ", noConfigError)
 		log.Fatal(noConfigError)
-		os.Exit(1)
 	}

 	if err := viper.ReadInConfig(); err == nil {
 		log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
 	} else {
 		invalidConfigError := "Couldn't read config file"
-		fmt.Println("Error: ", invalidConfigError)
-		log.Fatal(invalidConfigError)
-		os.Exit(1)
+		formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error())
+		log.Fatal(formattedError)
 	}
 }

@@ -132,7 +155,7 @@ func getBlockChain() *geth.BlockChain {
 	rawRpcClient, err := rpc.Dial(ipc)

 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
 	ethClient := ethclient.NewClient(rawRpcClient)
@@ -5,7 +5,8 @@ CREATE TABLE public.queued_storage (
  block_hash    BYTEA,
  contract      BYTEA,
  storage_key   BYTEA,
-  storage_value BYTEA
+  storage_value BYTEA,
+  UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
 );

 -- +goose Down
@@ -1,5 +1,5 @@
 -- +goose Up
-CREATE TABLE light_sync_transactions (
+CREATE TABLE header_sync_transactions (
  id          SERIAL PRIMARY KEY,
  header_id   INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
  hash        VARCHAR(66),
@@ -16,4 +16,4 @@ CREATE TABLE light_sync_transactions (
 );

 -- +goose Down
-DROP TABLE light_sync_transactions;
+DROP TABLE header_sync_transactions;
@@ -1,7 +1,7 @@
 -- +goose Up
-CREATE TABLE light_sync_receipts(
+CREATE TABLE header_sync_receipts(
  id                  SERIAL PRIMARY KEY,
-  transaction_id      INTEGER NOT NULL REFERENCES light_sync_transactions(id) ON DELETE CASCADE,
+  transaction_id      INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE,
  header_id           INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
  contract_address    VARCHAR(42),
  cumulative_gas_used NUMERIC,
@@ -15,4 +15,4 @@ CREATE TABLE light_sync_receipts(


 -- +goose Down
-DROP TABLE light_sync_receipts;
+DROP TABLE header_sync_receipts;
@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --

-- Dumped from database version 10.6
-- Dumped by pg_dump version 10.6
+-- Dumped from database version 11.3
+-- Dumped by pg_dump version 11.3

 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -12,23 +12,10 @@ SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
 SELECT pg_catalog.set_config('search_path', '', false);
 SET check_function_bodies = false;
+SET xmloption = content;
 SET client_min_messages = warning;
 SET row_security = off;

--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
 SET default_tablespace = '';

 SET default_with_oids = false;
@@ -259,6 +246,84 @@ CREATE SEQUENCE public.goose_db_version_id_seq
 ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;


+--
+-- Name: header_sync_receipts; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.header_sync_receipts (
+    id integer NOT NULL,
+    transaction_id integer NOT NULL,
+    header_id integer NOT NULL,
+    contract_address character varying(42),
+    cumulative_gas_used numeric,
+    gas_used numeric,
+    state_root character varying(66),
+    status integer,
+    tx_hash character varying(66),
+    rlp bytea
+);
+
+
+--
+-- Name: header_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.header_sync_receipts_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: header_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.header_sync_receipts_id_seq OWNED BY public.header_sync_receipts.id;
+
+
+--
+-- Name: header_sync_transactions; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.header_sync_transactions (
+    id integer NOT NULL,
+    header_id integer NOT NULL,
+    hash character varying(66),
+    gas_limit numeric,
+    gas_price numeric,
+    input_data bytea,
+    nonce numeric,
+    raw bytea,
+    tx_from character varying(44),
+    tx_index integer,
+    tx_to character varying(44),
+    value numeric
+);
+
+
+--
+-- Name: header_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public.header_sync_transactions_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: header_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public.header_sync_transactions_id_seq OWNED BY public.header_sync_transactions.id;
+
+
 --
 -- Name: headers; Type: TABLE; Schema: public; Owner: -
 --
@@ -294,84 +359,6 @@ CREATE SEQUENCE public.headers_id_seq
 ALTER SEQUENCE public.headers_id_seq OWNED BY public.headers.id;


--
-- Name: light_sync_receipts; Type: TABLE; Schema: public; Owner: -
--
-
-CREATE TABLE public.light_sync_receipts (
-    id integer NOT NULL,
-    transaction_id integer NOT NULL,
-    header_id integer NOT NULL,
-    contract_address character varying(42),
-    cumulative_gas_used numeric,
-    gas_used numeric,
-    state_root character varying(66),
-    status integer,
-    tx_hash character varying(66),
-    rlp bytea
-);
-
-
--
-- Name: light_sync_receipts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-
-CREATE SEQUENCE public.light_sync_receipts_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
--
-- Name: light_sync_receipts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-
-ALTER SEQUENCE public.light_sync_receipts_id_seq OWNED BY public.light_sync_receipts.id;
-
-
--
-- Name: light_sync_transactions; Type: TABLE; Schema: public; Owner: -
--
-
-CREATE TABLE public.light_sync_transactions (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    hash character varying(66),
-    gas_limit numeric,
-    gas_price numeric,
-    input_data bytea,
-    nonce numeric,
-    raw bytea,
-    tx_from character varying(44),
-    tx_index integer,
-    tx_to character varying(44),
-    value numeric
-);
-
-
--
-- Name: light_sync_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-
-CREATE SEQUENCE public.light_sync_transactions_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
--
-- Name: light_sync_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-
-ALTER SEQUENCE public.light_sync_transactions_id_seq OWNED BY public.light_sync_transactions.id;
-
-
 --
 -- Name: log_filters; Type: TABLE; Schema: public; Owner: -
 --
@@ -619,6 +606,20 @@ ALTER TABLE ONLY public.full_sync_transactions ALTER COLUMN id SET DEFAULT nextv
 ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);


+--
+-- Name: header_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.header_sync_receipts_id_seq'::regclass);
+
+
+--
+-- Name: header_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.header_sync_transactions_id_seq'::regclass);
+
+
 --
 -- Name: headers id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -626,20 +627,6 @@ ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('pu
 ALTER TABLE ONLY public.headers ALTER COLUMN id SET DEFAULT nextval('public.headers_id_seq'::regclass);


--
-- Name: light_sync_receipts id; Type: DEFAULT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_receipts ALTER COLUMN id SET DEFAULT nextval('public.light_sync_receipts_id_seq'::regclass);
-
-
--
-- Name: light_sync_transactions id; Type: DEFAULT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_transactions ALTER COLUMN id SET DEFAULT nextval('public.light_sync_transactions_id_seq'::regclass);
-
-
 --
 -- Name: log_filters id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -739,6 +726,38 @@ ALTER TABLE ONLY public.goose_db_version
    ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);


+--
+-- Name: header_sync_receipts header_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
+
+
+--
+-- Name: header_sync_receipts header_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: header_sync_transactions header_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions
+    ADD CONSTRAINT header_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
+
+
+--
+-- Name: header_sync_transactions header_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions
+    ADD CONSTRAINT header_sync_transactions_pkey PRIMARY KEY (id);
+
+
 --
 -- Name: headers headers_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -747,38 +766,6 @@ ALTER TABLE ONLY public.headers
    ADD CONSTRAINT headers_pkey PRIMARY KEY (id);


--
-- Name: light_sync_receipts light_sync_receipts_header_id_transaction_id_key; Type: CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_header_id_transaction_id_key UNIQUE (header_id, transaction_id);
-
-
--
-- Name: light_sync_receipts light_sync_receipts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_pkey PRIMARY KEY (id);
-
-
--
-- Name: light_sync_transactions light_sync_transactions_header_id_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_transactions
-    ADD CONSTRAINT light_sync_transactions_header_id_hash_key UNIQUE (header_id, hash);
-
-
--
-- Name: light_sync_transactions light_sync_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_transactions
-    ADD CONSTRAINT light_sync_transactions_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -803,6 +790,14 @@ ALTER TABLE ONLY public.eth_nodes
    ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);


+--
+-- Name: queued_storage queued_storage_block_height_block_hash_contract_storage_key_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.queued_storage
+    ADD CONSTRAINT queued_storage_block_height_block_hash_contract_storage_key_key UNIQUE (block_height, block_hash, contract, storage_key, storage_value);
+
+
 --
 -- Name: queued_storage queued_storage_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -909,6 +904,30 @@ ALTER TABLE ONLY public.full_sync_transactions
    ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;


+--
+-- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
+--
+-- Name: header_sync_receipts header_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_receipts
+    ADD CONSTRAINT header_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.header_sync_transactions(id) ON DELETE CASCADE;
+
+
+--
+-- Name: header_sync_transactions header_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.header_sync_transactions
+    ADD CONSTRAINT header_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
+
+
 --
 -- Name: headers headers_eth_node_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -917,30 +936,6 @@ ALTER TABLE ONLY public.headers
    ADD CONSTRAINT headers_eth_node_id_fkey FOREIGN KEY (eth_node_id) REFERENCES public.eth_nodes(id) ON DELETE CASCADE;


--
-- Name: light_sync_receipts light_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
--
-- Name: light_sync_receipts light_sync_receipts_transaction_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_receipts
-    ADD CONSTRAINT light_sync_receipts_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES public.light_sync_transactions(id) ON DELETE CASCADE;
-
-
--
-- Name: light_sync_transactions light_sync_transactions_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public.light_sync_transactions
-    ADD CONSTRAINT light_sync_transactions_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
-
-
 --
 -- Name: blocks node_fk; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -2,7 +2,7 @@ S
 `Dockerfile` will build an alpine image containing:
 - vDB as a binary with runtime deps statically linked: `/app/vulcanizedb`
 - The migration tool goose: `/app/goose`
- Two services for running `lightSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.
+- Two services for running `headerSync` and `continuousLogSync`, started with the default configuration `environments/staging.toml`.

 By default, vDB is configured towards the Kovan deploy. The configuration values can be overridden using environment variables, using the same hierarchical naming pattern but in CAPS and using underscores. For example, the contract address for the `Pit` can be set with the variable `CONTRACT_ADDRESS_PIT="0x123..."`.

@@ -18,8 +18,8 @@ e`
  * `CONTRACT_ADDRESS_[CONTRACT NAME]=0x123...`
  * `CONTRACT_ABI_[CONTRACT NAME]="ABI STRING"`
  * `CONTRACT_DEPLOYMENT-BLOCK_[CONTRACT NAME]=0` (doesn't really matter on a short chain, just avoids long unnecessary searching)
-6. Start the `lightSync` and `continuousLogSync` services:
-  * `./vulcanizedb lightSync --config environments/staging.toml`
+6. Start the `headerSync` and `continuousLogSync` services:
+  * `./vulcanizedb headerSync --config environments/staging.toml`
  * `./vulcanizedb continuousLogSync --config environments/staging.toml`

 ### Automated
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Runs the migrations and starts the lightSync and continuousLogSync services
+# Runs the migrations and starts the headerSync and continuousLogSync services

 # Exit if the variable tests fail
 set -e
@@ -21,7 +21,7 @@ set +e
 ./goose postgres "$CONNECT_STRING" up
 if [ $? -eq 0 ]; then
  # Fire up the services
-  ./vulcanizedb lightSync --config environments/staging.toml &
+  ./vulcanizedb headerSync --config environments/staging.toml &
  ./vulcanizedb continuousLogSync --config environments/staging.toml &
 else
  echo "Could not run migrations. Are the database details correct?"
@@ -1,11 +1,50 @@
 # Contribution guidelines

-Contributions are welcome! In addition to core contributions, developers are encouraged to build their own custom transformers which
-can be run together with other custom transformers using the [composeAndExeucte](../../staging/documentation/composeAndExecute.md) command.
+Contributions are welcome! Please open an Issues or Pull Request for any changes.
+
+In addition to core contributions, developers are encouraged to build their own custom transformers which
+can be run together with other custom transformers using the [composeAndExecute](../../staging/documentation/custom-transformers.md) command.
+
+## Pull Requests
+- `go fmt` is run as part of `make test` and `make integrationtest`, please make sure to check in the format changes.
+- Ensure that new code is well tested, including integration testing if applicable.
+- Make sure the build is passing.
+- Update the README or any [documentation files](./) as necessary. If editing the Readme, please
+conform to the
+[standard-readme specification](https://github.com/RichardLitt/standard-readme).
+- Once a Pull Request has received two approvals it can be merged in by a core developer.

 ## Creating a new migration file
 1. `make new_migration NAME=add_columnA_to_table1`
    - This will create a new timestamped migration file in `db/migrations`
 1. Write the migration code in the created file, under the respective `goose` pragma
    - Goose automatically runs each migration in a transaction; don't add `BEGIN` and `COMMIT` statements.
-1. Core migrations should be committed in their `goose fix`ed form.
+1. Core migrations should be committed in their `goose fix`ed form. To do this, run `make version_migrations` which
+converts timestamped migrations to migrations versioned by an incremented integer.
+
+## Diagrams
+- Diagrams were created with [draw.io](https://www.draw.io).
+- To update a diagram:
+  1. Go to [draw.io](https://www.draw.io).
+  1. Click on *File > Open from* and choose the location of the diagram you want to update.
+  1. Once open in draw.io, you may update it.
+  1. Export the diagram to this repository's directory and commit it.
+
+
+## Generating the Changelog
+We use [github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) to generate release Changelogs. To be consistent with previous Changelogs, the following flags should be passed to the command:
+
+```
+--user vulcanize
+--project vulcanizedb
+--token {YOUR_GITHUB_TOKEN}
+--no-issues
+--usernames-as-github-logins
+--since-tag {PREVIOUS_RELEASE_TAG}
+```
+
+For more information on why your github token is needed, and how to generate it see [https://github
+.com/github-changelog-generator/github-changelog-generator#github-token](https://github.com/github-changelog-generator/github-changelog-generator#github-token).
+
+## Code of Conduct
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
@@ -1,26 +1,30 @@
-# composeAndExecute
-The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers.
-This is accomplished by generating a Go pluggin which allows the `vulcanizedb` binary to link to external transformers, so
-long as they abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
+# Custom Transformers
+When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
+purpose can be leveraged.

-This command requires Go 1.11+ and [Go plugins](https://golang.org/pkg/plugin/) only work on Unix-based systems.
+Individual custom transformers can be composed together from any number of external repositories and executed as a
+single process using the `compose` and `execute` commands or the `composeAndExecute` command. This is accomplished by
+generating a Go plugin which allows the `vulcanizedb` binary to link to the external transformers, so long as they
+abide by one of the standard [interfaces](../staging/libraries/shared/transformer).

 ## Writing custom transformers
-Storage Transformers
+For help with writing different types of custom transformers please see below:
+
+Storage Transformers: transform data derived from contract storage tries
   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
-    
-Event Transformers   
+
+Event Transformers: transform data derived from Ethereum log events
   * [Guide](../../staging/libraries/shared/factories/event/README.md)
   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)

-Contract Transformers
+Contract Transformers: transform data derived from Ethereum log events and use it to poll public contract methods
   * [Example 1](https://github.com/vulcanize/account_transformers)
   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)

-## Preparing transformers to work as a plugin for composeAndExecute
+## Preparing custom transformers to work as part of a plugin
 To plug in an external transformer we need to:

 1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
@@ -35,16 +39,56 @@ or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) wa
    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
    as they are [here](../../staging/db/migrations)

-To update a plugin repository with changes to the core vulcanizedb repository, replace the vulcanizedb vendored in the plugin repo (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb`)
-with the newly updated version
-* The entire vendor lib within the vendored vulcanizedb needs to be deleted (`plugin_repo/vendor/github.com/vulcanize/vulcanizedb/vendor`)
-* These complications arise due to this [conflict](https://github.com/golang/go/issues/20481) between `dep` and Go plugins
+To update a plugin repository with changes to the core vulcanizedb repository, use your dependency manager to install the desired version of vDB.

-## Configuration
-A .toml config file is specified when executing the command:
-`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
+## Building and Running Custom Transformers
+### Commands
+* The `compose`, `execute`, `composeAndExecute` commands require Go 1.11+ and use [Go plugins](https://golang
+.org/pkg/plugin/) which only work on Unix-based systems.

-The config provides information for composing a set of transformers:
+* There is an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use of vendored
+dependencies which imposes certain limitations on how the plugins are built.
+
+* Separate `compose` and `execute` commands allow pre-building and linking to the pre-built .so file. So, if
+these are run independently, instead of using `composeAndExecute`, a couple of things need to be considered:
+    * It is necessary that the .so file was built with the same exact dependencies that are present in the execution
+    environment, i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
+    * The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so
+    in a different environment than the one it was composed in, then the database structure will need to be loaded 
+    into the environment's Postgres database. This can either be done by manually loading the plugin's schema into 
+    Postgres, or by manually running the plugin's migrations.
+     
+* The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's 
+`$GOPATH`, and that the plugin dependencies are present at their `$GOPATH` directories.
+
+* The `execute` command does not require the plugin transformer dependencies be located in their `$GOPATH` directories,
+instead it expects a .so file (of the name specified in the config file) to be in
+`$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin db migrations to
+ have already been ran against the database.
+
+ * Usage:
+     * compose: `./vulcanizedb compose --config=environments/config_name.toml`
+
+     * execute: `./vulcanizedb execute --config=environments/config_name.toml`
+
+     * composeAndExecute: `./vulcanizedb composeAndExecute --config=environments/config_name.toml`
+
+### Flags
+The `execute` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
+
+- `--recheck-headers`/`-r` - specifies whether to re-check headers for events after the header has already been queried for watched logs.
+Can be useful for redundancy if you suspect that your node is not always returning all desired logs on every query.
+Argument is expected to be a boolean: e.g. `-r=true`.
+Defaults to `false`.
+
+- `query-recheck-interval`/`-q` - specifies interval for re-checking storage diffs that haven been queued for later processing
+(by default, the storage watched queues storage diffs if transformer execution fails, on the assumption that subsequent data derived from the event transformers may enable us to decode storage keys that we don't recognize right now).
+Argument is expected to be a duration (integer measured in nanoseconds): e.g. `-q=10m30s` (for 10 minute, 30 second intervals).
+Defaults to `5m` (5 minutes).
+
+### Configuration
+A .toml config file is specified when executing the commands.
+The config provides information for composing a set of transformers from external repositories:

 ```toml
 [database]
@@ -107,7 +151,7 @@ The config provides information for composing a set of transformers:
         that fetches event logs from an ETH node
        - `eth_contract` indicates the transformer works with the [contract watcher](../staging/libraries/shared/watcher/contract_watcher.go)
        that is made to work with [contract_watcher pkg](../../staging/pkg/contract_watcher)
-        based transformers which work with either a light or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
+        based transformers which work with either a header or full sync vDB to watch events and poll public methods ([example1](https://github.com/vulcanize/account_transformers/tree/master/transformers/account/light), [example2](https://github.com/vulcanize/ens_transformers/tree/working/transformers/domain_records))
    - `migrations` is the relative path from `repository` to the db migrations directory for the transformer
    - `rank` determines the order that migrations are ran, with lower ranked migrations running first
        - this is to help isolate any potential conflicts between transformer migrations
@@ -0,0 +1,81 @@
+# Syncing commands
+These commands are used to sync raw Ethereum data into Postgres, with varying levels of data granularity.
+
+## headerSync
+Syncs block headers from a running Ethereum node into the VulcanizeDB table `headers`.
+- Queries the Ethereum node using RPC calls.
+- Validates headers from the last 15 blocks to ensure that data is up to date.
+- Useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart
+contract storage values or event logs).
+- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
+different from what we have already stored in the database, the header record will be updated.
+
+#### Usage
+- Run: `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
+- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    ipcPath  = <path to a running Ethereum node>
+```
+- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
+
+## fullSync
+Syncs blocks, transactions, receipts and logs from a running Ethereum node into VulcanizeDB tables named
+`blocks`, `uncles`, `full_sync_transactions`, `full_sync_receipts` and `logs`. 
+- Queries the Ethereum node using RPC calls.
+- Validates headers from the last 15 blocks to ensure that data is up to date.
+- Useful when you want to maintain a broad cache of what's happening on the blockchain.
+- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
+different from what we have already stored in the database, the header record will be updated.
+
+#### Usage
+- Run `./vulcanizedb fullSync --config <config.toml> --starting-block-number <block-number>`
+- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    ipcPath  = <path to a running Ethereum node>
+```
+- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
+
+*Please note, that if you are fast syncing your Ethereum node, wait for the initial sync to finish.*
+
+## coldImport
+Syncs VulcanizeDB from Geth's underlying LevelDB datastore and persists Ethereum blocks, 
+transactions, receipts and logs into VulcanizeDB tables named `blocks`, `uncles`, 
+`full_sync_transactions`, `full_sync_receipts` and `logs` respectively.
+
+#### Usage
+1. Ensure the Ethereum node you're point at is not running, and that it has synced to the desired block height.
+1. Run `./vulcanizedb coldImport --config <config.toml>`
+1. Optional flags:
+    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
+    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
+    - `--all`/`-a`: sync all missing blocks
+
+The config file can be formatted as follows, and must contain the LevelDB path.
+
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    leveldbpath = "/Users/user/Library/Ethereum/geth/chaindata"
+```
@@ -1,15 +1,15 @@
-# contractWatcher
+# Generic Transformer
 The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available.
 It also provides some state variable coverage by automating polling of public methods, with some restrictions:
 1. The method must have 2 or less arguments
 1. The method's arguments must all be of type address or bytes32 (hash)
 1. The method must return a single value

-This command operates in two modes- `light` and `full`- which require a light or full-synced vulcanizeDB, respectively.
+This command operates in two modes- `header` and `full`- which require a header or full-synced vulcanizeDB, respectively.

 This command requires the contract ABI be available on Etherscan if it is not provided in the config file by the user.

-If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `light` or `full` mode.
+If method polling is turned on we require an archival node at the ETH ipc endpoint in our config, whether or not we are operating in `header` or `full` mode.
 Otherwise we only need to connect to a full node.

 ## Configuration
@@ -97,16 +97,16 @@ Modify `./environments/example.toml` to replace the empty `ipcPath` with a path
 This endpoint should be for an archival eth node if we want to perform method polling as this configuration is currently set up to do. To work with a non-archival full node,
 remove the `balanceOf` method from the `0x8dd5fbce2f6a956c3022ba3663759011dd51e73e` (TrueUSD) contract.

-If you are operating a light sync vDB, run:
+If you are operating a header sync vDB, run:

- `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=light`
+ `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=header`

 If instead you are operating a full sync vDB and provided an archival node IPC path, run in full mode:

 `./vulcanizedb contractWatcher --config=./environments/example.toml --mode=full`

 This will run the contractWatcher and configures it to watch the contracts specified in the config file. Note that
-by default we operate in `light` mode but the flag is included here to demonstrate its use.
+by default we operate in `header` mode but the flag is included here to demonstrate its use.

 The example config we link to in this example watches two contracts, the ENS Registry (0x314159265dD8dbb310642f98f50C066173C1259b) and TrueUSD (0x8dd5fbCe2F6a956C3022bA3663759011Dd51e73E).

@@ -117,43 +117,43 @@ The TrueUSD contract is configured with two events (`Transfer` and `Mint`) and a
 to poll the `balanceOf` method with those addresses at every block. Note that we do not provide an ABI for TrueUSD as its ABI can be fetched from Etherscan.

 For the ENS contract, it produces and populates a schema with four tables"
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
-`light_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.newowner_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.newresolver_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.newttl_event`
+`header_0x314159265dd8dbb310642f98f50c066173c1259b.transfer_event`

 For the TrusUSD contract, it produces and populates a schema with three tables:

-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
-`light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`
+`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event`
+`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.mint_event`
+`header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method`

 Column ids and types for these tables are generated based on the event and method argument names and types and method return types, resulting in tables such as:

-Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"
+Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event"

-|  Column    |         Type          | Collation | Nullable |                                           Default                                           | Storage  | Stats target | Description  
-|:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
-| id         | integer               |           | not null | nextval('light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain    |              |             |
-| header_id  | integer               |           | not null |                                                                                             | plain    |              |             |
-| token_name | character varying(66) |           | not null |                                                                                             | extended |              |             |
-| raw_log    | jsonb                 |           |          |                                                                                             | extended |              |             |
-| log_idx    | integer               |           | not null |                                                                                             | plain    |              |             |
-| tx_idx     | integer               |           | not null |                                                                                             | plain    |              |             |
-| from_      | character varying(66) |           | not null |                                                                                             | extended |              |             |
-| to_        | character varying(66) |           | not null |                                                                                             | extended |              |             |
-| value_     | numeric               |           | not null |                                                                                             | main     |              |             |
+|  Column    |         Type          | Collation | Nullable |                                           Default                                            | Storage  | Stats target | Description  
+|:----------:|:---------------------:|:---------:|:--------:|:--------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
+| id         | integer               |           | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.transfer_event_id_seq'::regclass) | plain    |              |             |
+| header_id  | integer               |           | not null |                                                                                              | plain    |              |             |
+| token_name | character varying(66) |           | not null |                                                                                              | extended |              |             |
+| raw_log    | jsonb                 |           |          |                                                                                              | extended |              |             |
+| log_idx    | integer               |           | not null |                                                                                              | plain    |              |             |
+| tx_idx     | integer               |           | not null |                                                                                              | plain    |              |             |
+| from_      | character varying(66) |           | not null |                                                                                              | extended |              |             |
+| to_        | character varying(66) |           | not null |                                                                                              | extended |              |             |
+| value_     | numeric               |           | not null |                                                                                              | main     |              |             |


-Table "light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"
+Table "header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method"

-|   Column   |         Type          | Collation | Nullable |                                            Default                                            | Storage  | Stats target | Description |
-|:----------:|:---------------------:|:---------:|:--------:|:-------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
-| id         | integer               |           | not null | nextval('light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain    |              |             |
-| token_name | character varying(66) |           | not null |                                                                                               | extended |              |             |
-| block      | integer               |           | not null |                                                                                               | plain    |              |             |
-| who_       | character varying(66) |           | not null |                                                                                               | extended |              |             |
-| returned   | numeric               |           | not null |                                                                                               | main     |              |             |
+|   Column   |         Type          | Collation | Nullable |                                            Default                                             | Storage  | Stats target | Description |
+|:----------:|:---------------------:|:---------:|:--------:|:----------------------------------------------------------------------------------------------:|:--------:|:------------:|:-----------:|
+| id         | integer               |           | not null | nextval('header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e.balanceof_method_id_seq'::regclass) | plain    |              |             |
+| token_name | character varying(66) |           | not null |                                                                                                | extended |              |             |
+| block      | integer               |           | not null |                                                                                                | plain    |              |             |
+| who_       | character varying(66) |           | not null |                                                                                                | extended |              |             |
+| returned   | numeric               |           | not null |                                                                                                | main     |              |             |
  
 The addition of '_' after table names is to prevent collisions with reserved Postgres words.

@@ -0,0 +1,34 @@
+# Postgraphile
+
+You can expose VulcanizeDB data via [Postgraphile](https://github.com/graphile/postgraphile).
+Check out [their documentation](https://www.graphile.org/postgraphile/) for the most up-to-date instructions on installing, running, and customizing Postgraphile.
+
+## Simple Setup
+
+As of April 30, 2019, you can run Postgraphile pointed at the default `vulcanize_public` database with the following commands:
+
+```
+npm install -g postgraphile
+postgraphile --connection postgres://localhost/vulcanize_public --schema=public,custom --disable-default-mutations --no-ignore-rbac
+```
+
+Arguments:
+- `--connection` specifies the database. The above command connects to the default `vulcanize_public` database 
+defined in [the example config](../environments/public.toml.example).
+- `--schema` defines what schema(s) to expose. The above exposes the `public` schema (for core VulcanizeDB data) as well as a `custom` schema (where `custom` is the name of a schema defined in executed transformers).
+- `--disable-default-mutations` prevents Postgraphile from exposing create, update, and delete operations on your data, which are otherwise enabled by default.
+- `--no-ignore-rbac` ensures that Postgraphile will only expose the tables, columns, fields, and query functions that the user has explicit access to.
+
+## Customizing Postgraphile
+
+By default, Postgraphile will expose queries for all tables defined in your chosen database/schema(s), including [filtering](https://www.graphile.org/postgraphile/filtering/) and [auto-discovered relations](https://www.graphile.org/postgraphile/relations/).
+
+If you'd like to expose more customized windows into your data, there are some techniques you can apply when writing migrations:
+
+- [Computed columns](https://www.graphile.org/postgraphile/computed-columns/) enable you to derive additional fields from types defined in your database.
+For example, you could write a function to expose a block header's state root over Postgraphile with a computed column - without modifying the `public.headers` table.
+- [Custom queries](https://www.graphile.org/postgraphile/custom-queries/) enable you to provide on-demand access to more complex data (e.g. the product of joining and filtering several tables' data based on a passed argument).
+For example, you could write a custom query to get the block timestamp for every transaction originating from a given address.
+- [Subscriptions](https://www.graphile.org/postgraphile/subscriptions/) enable you to publish data as it is coming into your database.
+
+The above list is not exhaustive - please see the Postgraphile documentation for a more comprehensive and up-to-date description of available features.
@@ -1,26 +0,0 @@
-# Syncing commands
-These commands are used to sync raw Ethereum data into Postgres.
-
-## lightSync
-Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
-This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values or event logs).
-1. Start Ethereum node
-1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb lightSync --config <config.toml> --starting-block-number <block-number>`
-    
-## sync
-Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.
-This command is useful when you want to maintain a broad cache of what's happening on the blockchain.
-1. Start Ethereum node (**if fast syncing your Ethereum node, wait for initial sync to finish**)
-1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb sync --config <config.toml> --starting-block-number <block-number>`
-
-## coldImport
-Sync VulcanizeDB from the LevelDB underlying a Geth node.
-1. Assure node is not running, and that it has synced to the desired block height.
-1. Start vulcanize_db
-   - `./vulcanizedb coldImport --config <config.toml>`
-1. Optional flags:
-    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
-    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
-    - `--all`/`-a`: sync all missing blocks
@@ -0,0 +1,65 @@
+module github.com/vulcanize/vulcanizedb
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/allegro/bigcache v1.1.0 // indirect
+	github.com/apilayer/freegeoip v3.5.0+incompatible // indirect
+	github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76 // indirect
+	github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc // indirect
+	github.com/cespare/cp v1.1.1 // indirect
+	github.com/dave/jennifer v1.3.0
+	github.com/deckarep/golang-set v1.7.1 // indirect
+	github.com/docker/docker v1.13.1 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/elastic/gosigar v0.10.4 // indirect
+	github.com/ethereum/go-ethereum v1.9.0
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/google/uuid v1.0.0 // indirect
+	github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598 // indirect
+	github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47
+	github.com/howeyc/fsnotify v0.9.0 // indirect
+	github.com/hpcloud/tail v1.0.0
+	github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/influxdata/influxdb v1.7.7 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
+	github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0
+	github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0 // indirect
+	github.com/lib/pq v1.0.0
+	github.com/mattn/go-colorable v0.1.2 // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/mattn/go-sqlite3 v1.11.0 // indirect
+	github.com/mitchellh/go-homedir v1.0.0
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/olekukonko/tablewriter v0.0.1 // indirect
+	github.com/onsi/ginkgo v1.6.0
+	github.com/onsi/gomega v1.4.2
+	github.com/oschwald/maxminddb-golang v1.3.1 // indirect
+	github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/pressly/goose v2.6.0+incompatible
+	github.com/prometheus/tsdb v0.9.1 // indirect
+	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rs/cors v1.5.0 // indirect
+	github.com/sirupsen/logrus v1.2.0
+	github.com/spf13/cobra v0.0.3
+	github.com/spf13/viper v1.2.0
+	github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect
+	github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f // indirect
+	github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2 // indirect
+	github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d // indirect
+	github.com/tyler-smith/go-bip39 v1.0.0 // indirect
+	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
+	golang.org/x/net v0.0.0-20190603091049-60506f45cf65
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	google.golang.org/appengine v1.6.1 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+)
@@ -0,0 +1,226 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/allegro/bigcache v1.1.0 h1:MLuIKTjdxDc+qsG2rhjsYjsHQC5LUGjIWzutg7M+W68=
+github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A=
+github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU=
+github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76 h1:64W/KrGykPTfDI9xTkZtnjZRYA5p2+c/IuGgjzeWCpI=
+github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc h1:nLRj+ULuRYb0qTAOnuayFXRnLjYXBots5CSp5zH4RqU=
+github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
+github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
+github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
+github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
+github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo=
+github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
+github.com/ethereum/go-ethereum v1.9.0 h1:9Kaf7UfDkV3aIUJlf14hI/GgEgRAUq60u4fBlb9dLWw=
+github.com/ethereum/go-ethereum v1.9.0/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598 h1:XLoCW/kXxbvPvp216Kq/c+TtwWYHy9sjeDidFcG45g0=
+github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
+github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324 h1:PV190X5/DzQ/tbFFG5YpT5mH6q+cHlfgqI5JuRnH9oE=
+github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb v1.7.7 h1:UvNzAPfBrKMENVbQ4mr4ccA9sW+W1Ihl0Yh1s0BiVAg=
+github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
+github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE=
+github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0 h1:S8kWZLXHpcOq3nGAvIs0oDgd4CXxkxE3hkDVRjTu7ro=
+github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I=
+github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo=
+github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
+github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.9.1 h1:IWaAmWkYlgG7/S4iw4IpAQt5Y35QaZM6/GsZ7GsjAuk=
+github.com/prometheus/tsdb v0.9.1/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
+github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
+github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
+github.com/rs/cors v1.5.0 h1:dgSHE6+ia18arGOTIYQKKGWLvEbGvmbNE6NfxhoNHUY=
+github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.2.0 h1:M4Rzxlu+RgU4pyBRKhKaVN1VeYOm8h2jgyXnAseDgCc=
+github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
+github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f h1:T7YHzO3/eqD/kv5m9+TLM4XuEAkN7NPj5pnZHqaOo/Q=
+github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
+github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2 h1:o6NMd68tuqfQ0ZFnz2d16xzFNLWxrCvqF40InOJJHSM=
+github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc=
+github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/tyler-smith/go-bip39 v1.0.0 h1:FOHg9gaQLeBBRbHE/QrTLfEiBHy5pQ/yXzf9JG5pYFM=
+github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
+github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
+github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -286,7 +286,7 @@ var _ = Describe("contractWatcher full transformer", func() {
 			err = t.Execute()
 			Expect(err).ToNot(HaveOccurred())

-			log := test_helpers.LightNewOwnerLog{}
+			log := test_helpers.HeaderSyncNewOwnerLog{}
 			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM full_%s.newowner_event", ensAddr)).StructScan(&log)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("does not exist"))
@@ -9,7 +9,7 @@ import (
 	. "github.com/onsi/gomega"

 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
@@ -18,7 +18,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )

-var _ = Describe("contractWatcher light transformer", func() {
+var _ = Describe("contractWatcher headerSync transformer", func() {
 	var db *postgres.DB
 	var err error
 	var blockChain core.BlockChain
@@ -49,6 +49,13 @@ var _ = Describe("contractWatcher light transformer", func() {
 			c, ok := t.Contracts[tusdAddr]
 			Expect(ok).To(Equal(true))

+			// TODO: Fix this
+			// This test sometimes randomly fails because
+			// for some reason the starting block number is not updated from
+			// its original value (5197514) to the block number (6194632)
+			// of the earliest header (mocks.MockHeader1) in the repository
+			// It is not clear how this happens without one of the above insertErrs
+			// having been thrown and without any errors thrown during the Init() call
 			Expect(c.StartingBlock).To(Equal(int64(6194632)))
 			Expect(c.Abi).To(Equal(constants.TusdAbiString))
 			Expect(c.Name).To(Equal("TrueUSD"))
@@ -100,8 +107,8 @@ var _ = Describe("contractWatcher light transformer", func() {
 			err = t.Execute()
 			Expect(err).ToNot(HaveOccurred())

-			log := test_helpers.LightTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&log)
+			log := test_helpers.HeaderSyncTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&log)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(log.HeaderID).To(Equal(headerID))
@@ -168,12 +175,12 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())

 			res := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x1062a747393198f70F71ec65A582423Dba7E5Ab3' AND block = '6791669'", tusdAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Balance).To(Equal("55849938025000000000000"))
 			Expect(res.TokenName).To(Equal("TrueUSD"))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6791669'", tusdAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -208,8 +215,8 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(t.Start).To(Equal(int64(6885698)))

-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			log := test_helpers.HeaderSyncNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(log.HeaderID).To(Equal(headerID))
@@ -260,17 +267,17 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())

 			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 			Expect(res.TokenName).To(Equal(""))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Address).To(Equal("0x0000000000000000000000000000000000000000"))
 			Expect(res.TokenName).To(Equal(""))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x9THIS110dcc444fIS242510c09bbAbe21aFAKEcacNODE82f7b843HASH61ba391' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -287,8 +294,8 @@ var _ = Describe("contractWatcher light transformer", func() {
 			err = t.Execute()
 			Expect(err).ToNot(HaveOccurred())

-			log := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&log)
+			log := test_helpers.HeaderSyncNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&log)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("does not exist"))
 		})
@@ -309,12 +316,12 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())

 			res := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(res.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 			Expect(res.TokenName).To(Equal(""))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&res)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -338,16 +345,16 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(t.Start).To(Equal(int64(6885702)))

-			newOwnerLog := test_helpers.LightNewOwnerLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(newOwnerLog.Node).To(Equal("0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae"))
 			Expect(newOwnerLog.Label).To(Equal("0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047"))
 			Expect(newOwnerLog.Owner).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))

-			transferLog := test_helpers.LightTransferLog{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			transferLog := test_helpers.HeaderSyncTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
 			Expect(err).ToNot(HaveOccurred())
 			// We don't know vulcID, so compare individual fields instead of complete structures
 			Expect(transferLog.From).To(Equal("0x8cA465764873E71CEa525F5EB6AE973d650c22C2"))
@@ -410,27 +417,27 @@ var _ = Describe("contractWatcher light transformer", func() {
 			Expect(err).ToNot(HaveOccurred())

 			owner := test_helpers.Owner{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885696'", ensAddr)).StructScan(&owner)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(owner.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 			Expect(owner.TokenName).To(Equal(""))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ce695797aaf402b1c186bad9eca28842625b5047' AND block = '6885696'", ensAddr)).StructScan(&owner)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(owner.Address).To(Equal("0x0000000000000000000000000000000000000000"))
 			Expect(owner.TokenName).To(Equal(""))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x95832c7a47ff8a7840e28b78ceMADEUPaaf4HASHc186badTHItransformers.8IS625bFAKE' AND block = '6885696'", ensAddr)).StructScan(&owner)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))

 			bal := test_helpers.BalanceOf{}
-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x8cA465764873E71CEa525F5EB6AE973d650c22C2' AND block = '6885701'", tusdAddr)).StructScan(&bal)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(bal.Balance).To(Equal("1954436000000000000000"))
 			Expect(bal.TokenName).To(Equal("TrueUSD"))

-			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x09BbBBE21a5975cAc061D82f7b843b1234567890' AND block = '6885701'", tusdAddr)).StructScan(&bal)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no rows in result set"))
 		})
@@ -130,10 +130,10 @@ var _ = Describe("Poller", func() {
 		})
 	})

-	Describe("Light sync mode", func() {
+	Describe("Header sync mode", func() {
 		BeforeEach(func() {
 			db, bc = test_helpers.SetupDBandBC()
-			contractPoller = poller.NewPoller(bc, db, types.LightSync)
+			contractPoller = poller.NewPoller(bc, db, types.HeaderSync)
 		})

 		Describe("PollContract", func() {
@@ -148,22 +148,22 @@ var _ = Describe("Poller", func() {

 				scanStruct := test_helpers.BalanceOf{}

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("66386309548896882859581786"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0x3f5CE5FBFe3E9af3971dD833D26bA9b5C936f0bE' AND block = '6707323'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Balance).To(Equal("17982350181394112023885864"))
 				Expect(scanStruct.TokenName).To(Equal("TrueUSD"))
@@ -180,12 +180,12 @@ var _ = Describe("Poller", func() {

 				scanStruct := test_helpers.Owner{}

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x7e74a86b6e146964fb965db04dc2590516da77f720bb6759337bf5632415fd86' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x546aA2EaE2514494EeaDb7bbb35243348983C59d"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.owner_method WHERE node_ = '0x93cdeb708b7545dc668eb9280176169d1c33cfd8ed6f04690a0bcc88a93fc4ae' AND block = '6885877'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x6090A6e47849629b7245Dfa1Ca21D94cd15878Ef"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@@ -202,7 +202,7 @@ var _ = Describe("Poller", func() {

 				scanStruct := test_helpers.BalanceOf{}

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method WHERE who_ = '0xfE9e8709d3215310075d67E3ed32A380CCf451C8' AND block = '6707322'", constants.TusdContractAddress)).StructScan(&scanStruct)
 				Expect(err).To(HaveOccurred())
 			})

@@ -217,7 +217,7 @@ var _ = Describe("Poller", func() {
 				Expect(err).ToNot(HaveOccurred())

 				scanStruct := test_helpers.Resolver{}
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@@ -225,13 +225,13 @@ var _ = Describe("Poller", func() {

 				test_helpers.TearDown(db)
 				db, bc = test_helpers.SetupDBandBC()
-				contractPoller = poller.NewPoller(bc, db, types.LightSync)
+				contractPoller = poller.NewPoller(bc, db, types.HeaderSync)

 				con.Piping = true
 				err = contractPoller.PollContract(*con, 6921968)
 				Expect(err).ToNot(HaveOccurred())

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.resolver_method WHERE node_ = '0x495b6e6efdedb750aa519919b5cf282bdaa86067b82a2293a3ff5723527141e8' AND block = '6921967'", constants.EnsContractAddress)).StructScan(&scanStruct)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanStruct.Address).To(Equal("0x5FfC014343cd971B7eb70732021E26C35B744cc4"))
 				Expect(scanStruct.TokenName).To(Equal("ENS-Registry"))
@@ -2,13 +2,13 @@
 	
 One approach VulcanizeDB takes to caching and indexing smart contracts is to watch contract events emitted in receipt logs.

-With a light synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
+With a header synced vDB we can watch events by iterating over headers retrieved from the synced `headers` table and using these headers to
 fetch and verify relevant event logs from a full Ethereum node, keeping track of which headers we have checked for which events 
 with our `checked_headers` table.

 ## Assumptions

-This approach assumes you are running a vDB light sync which is itself run against a light Ethereum node,
+This approach assumes you are running a vDB header sync which is run against a light Ethereum node;
 this approach also assumes there is a full node available. 

 Looking forward, we will be building fetchers that enable sourcing data from IPFS instead of an ETH node.
@@ -17,7 +17,7 @@ Looking forward, we will be building fetchers that enable sourcing data from IPF

 VulcanizeDB has shared code built out for building and plugging in event transformers

-### [Event Watcher (light sync)](../staging/libraries/shared/watcher/event_watcher.go)
+### [Event Watcher (header sync)](../staging/libraries/shared/watcher/event_watcher.go)

 The event watcher is responsible for continuously fetching and delegating chunks of logs and their associated header to the appropriate transformers.

@@ -1,73 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event
-
-import (
-	"github.com/ethereum/go-ethereum/core/types"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-)
-
-type LogNoteTransformer struct {
-	Config     transformer.EventTransformerConfig
-	Converter  LogNoteConverter
-	Repository Repository
-}
-
-func (tr LogNoteTransformer) NewLogNoteTransformer(db *postgres.DB) transformer.EventTransformer {
-	tr.Repository.SetDB(db)
-	return tr
-}
-
-func (tr LogNoteTransformer) Execute(logs []types.Log, header core.Header, recheckedHeader constants.TransformerExecution) error {
-	transformerName := tr.Config.TransformerName
-
-	// No matching logs, mark the header as checked for this type of logs
-	if len(logs) < 1 {
-		err := tr.Repository.MarkHeaderChecked(header.Id)
-		if err != nil {
-			log.Printf("Error marking header as checked in %v: %v", transformerName, err)
-			return err
-		}
-		return nil
-	}
-
-	models, err := tr.Converter.ToModels(logs)
-	if err != nil {
-		log.Printf("Error converting logs in %v: %v", transformerName, err)
-		return err
-	}
-
-	err = tr.Repository.Create(header.Id, models)
-	if err != nil {
-		log.Printf("Error persisting %v record: %v", transformerName, err)
-		return err
-	}
-	return nil
-}
-
-func (tr LogNoteTransformer) GetName() string {
-	return tr.Config.TransformerName
-}
-
-func (tr LogNoteTransformer) GetConfig() transformer.EventTransformerConfig {
-	return tr.Config
-}
@@ -1,126 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package event_test
-
-import (
-	"math/rand"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/fakes"
-)
-
-var _ = Describe("LogNoteTransformer", func() {
-	var (
-		repository mocks.MockRepository
-		converter  mocks.MockLogNoteConverter
-		headerOne  core.Header
-		t          transformer.EventTransformer
-		model      test_data.GenericModel
-		config     = test_data.GenericTestConfig
-		logs       = test_data.GenericTestLogs
-	)
-
-	BeforeEach(func() {
-		repository = mocks.MockRepository{}
-		converter = mocks.MockLogNoteConverter{}
-		t = event.LogNoteTransformer{
-			Config:     config,
-			Converter:  &converter,
-			Repository: &repository,
-		}.NewLogNoteTransformer(nil)
-
-		headerOne = core.Header{Id: rand.Int63(), BlockNumber: rand.Int63()}
-	})
-
-	It("sets the database", func() {
-		Expect(repository.SetDbCalled).To(BeTrue())
-	})
-
-	It("marks header checked if no logs are provided", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
-	})
-
-	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.ToModelsCalledCounter).To(Equal(0))
-		Expect(repository.CreateCalledCounter).To(Equal(0))
-	})
-
-	It("does not call repository.MarkCheckedHeader when there are logs", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		repository.AssertMarkHeaderCheckedNotCalled()
-	})
-
-	It("returns error if marking header checked returns err", func() {
-		repository.SetMarkHeaderCheckedError(fakes.FakeError)
-
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("converts matching logs to models", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(converter.PassedLogs).To(Equal(logs))
-	})
-
-	It("returns error if converter returns error", func() {
-		converter.SetConverterError(fakes.FakeError)
-
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-
-	It("persists the model", func() {
-		converter.SetReturnModels([]interface{}{model})
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
-		Expect(repository.PassedModels).To(Equal([]interface{}{model}))
-	})
-
-	It("returns error if repository returns error for create", func() {
-		repository.SetCreateError(fakes.FakeError)
-
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
-
-		Expect(err).To(HaveOccurred())
-		Expect(err).To(MatchError(fakes.FakeError))
-	})
-})
@@ -17,14 +17,11 @@
 package event

 import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )

 type Repository interface {
 	Create(headerID int64, models []interface{}) error
 	MarkHeaderChecked(headerID int64) error
-	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
-	RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
 	SetDB(db *postgres.DB)
 }
@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	log "github.com/sirupsen/logrus"

-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -37,7 +36,7 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.Event
 	return transformer
 }

-func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+func (transformer Transformer) Execute(logs []types.Log, header core.Header) error {
 	transformerName := transformer.Config.TransformerName
 	config := transformer.Config

@@ -23,7 +23,6 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
@@ -60,14 +59,14 @@ var _ = Describe("Transformer", func() {
 	})

 	It("marks header checked if no logs returned", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]types.Log{}, headerOne)

 		Expect(err).NotTo(HaveOccurred())
 		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
 	})

 	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]types.Log{}, headerOne)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@@ -76,7 +75,7 @@ var _ = Describe("Transformer", func() {
 	})

 	It("does not call repository.MarkCheckedHeader when there are logs", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).NotTo(HaveOccurred())
 		repository.AssertMarkHeaderCheckedNotCalled()
@@ -85,14 +84,14 @@ var _ = Describe("Transformer", func() {
 	It("returns error if marking header checked returns err", func() {
 		repository.SetMarkHeaderCheckedError(fakes.FakeError)

-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]types.Log{}, headerOne)

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})

 	It("converts an eth log to an entity", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@@ -102,7 +101,7 @@ var _ = Describe("Transformer", func() {
 	It("returns an error if converter fails", func() {
 		converter.ToEntitiesError = fakes.FakeError

-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -111,7 +110,7 @@ var _ = Describe("Transformer", func() {
 	It("converts an entity to a model", func() {
 		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}

-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
@@ -121,7 +120,7 @@ var _ = Describe("Transformer", func() {
 		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
 		converter.ToModelsError = fakes.FakeError

-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -130,7 +129,7 @@ var _ = Describe("Transformer", func() {
 	It("persists the record", func() {
 		converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}

-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@@ -139,7 +138,7 @@ var _ = Describe("Transformer", func() {

 	It("returns error if persisting the record fails", func() {
 		repository.SetCreateError(fakes.FakeError)
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -100,6 +100,7 @@ type Repository interface {
 A contract-specific implementation of the repository interface enables the transformer to write the decoded storage value to the appropriate table in postgres.

 The `Create` function is expected to recognize and persist a given storage value by the variable's name, as indicated on the row's metadata.
+Note: we advise silently discarding duplicates in `Create` - as it's possible that you may read the same diff several times, and an error will trigger the storage watcher to queue that diff for later processing.

 The `SetDB` function is required for the repository to connect to the database.

@@ -100,4 +100,54 @@ var _ = Describe("Storage transformer", func() {
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
+
+	Describe("when a storage row contains more than one item packed in storage", func() {
+		var (
+			rawValue        = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
+			fakeBlockNumber = 123
+			fakeBlockHash   = "0x67890"
+			packedTypes     = make(map[int]utils.ValueType)
+		)
+		packedTypes[0] = utils.Uint48
+		packedTypes[1] = utils.Uint48
+
+		var fakeMetadata = utils.StorageValueMetadata{
+			Name:        "",
+			Keys:        nil,
+			Type:        utils.PackedSlot,
+			PackedTypes: packedTypes,
+		}
+
+		It("passes the decoded data items to the repository", func() {
+			mappings.Metadata = fakeMetadata
+			fakeRow := utils.StorageDiffRow{
+				Contract:     common.Address{},
+				BlockHash:    common.HexToHash(fakeBlockHash),
+				BlockHeight:  fakeBlockNumber,
+				StorageKey:   common.Hash{},
+				StorageValue: rawValue.Hash(),
+			}
+
+			err := t.Execute(fakeRow)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
+			Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
+			Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
+			expectedPassedValue := make(map[int]string)
+			expectedPassedValue[0] = "10800"
+			expectedPassedValue[1] = "172800"
+			Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
+		})
+
+		It("returns error if creating a row fails", func() {
+			mappings.Metadata = fakeMetadata
+			repository.CreateErr = fakes.FakeError
+
+			err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
 })
@@ -24,22 +24,22 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )

-type LogFetcher interface {
+type ILogFetcher interface {
 	FetchLogs(contractAddresses []common.Address, topics []common.Hash, missingHeader core.Header) ([]types.Log, error)
 }

-type Fetcher struct {
+type LogFetcher struct {
 	blockChain core.BlockChain
 }

-func NewFetcher(blockchain core.BlockChain) *Fetcher {
-	return &Fetcher{
+func NewLogFetcher(blockchain core.BlockChain) *LogFetcher {
+	return &LogFetcher{
 		blockChain: blockchain,
 	}
 }

 // Checks all topic0s, on all addresses, fetching matching logs for the given header
-func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
+func (logFetcher LogFetcher) FetchLogs(addresses []common.Address, topic0s []common.Hash, header core.Header) ([]types.Log, error) {
 	blockHash := common.HexToHash(header.Hash)
 	query := ethereum.FilterQuery{
 		BlockHash: &blockHash,
@@ -48,7 +48,7 @@ func (fetcher Fetcher) FetchLogs(addresses []common.Address, topic0s []common.Ha
 		Topics: [][]common.Hash{topic0s},
 	}

-	logs, err := fetcher.blockChain.GetEthLogsWithCustomQuery(query)
+	logs, err := logFetcher.blockChain.GetEthLogsWithCustomQuery(query)
 	if err != nil {
 		// TODO review aggregate fetching error handling
 		return []types.Log{}, err
@@ -22,16 +22,16 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	fetch "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 )

-var _ = Describe("Fetcher", func() {
+var _ = Describe("LogFetcher", func() {
 	Describe("FetchLogs", func() {
 		It("fetches logs based on the given query", func() {
 			blockChain := fakes.NewMockBlockChain()
-			fetcher := fetch.NewFetcher(blockChain)
+			logFetcher := fetcher.NewLogFetcher(blockChain)
 			header := fakes.FakeHeader

 			addresses := []common.Address{
@@ -41,7 +41,7 @@ var _ = Describe("Fetcher", func() {

 			topicZeros := []common.Hash{common.BytesToHash([]byte{1, 2, 3, 4, 5})}

-			_, err := fetcher.FetchLogs(addresses, topicZeros, header)
+			_, err := logFetcher.FetchLogs(addresses, topicZeros, header)

 			address1 := common.HexToAddress("0xfakeAddress")
 			address2 := common.HexToAddress("0xanotherFakeAddress")
@@ -59,9 +59,9 @@ var _ = Describe("Fetcher", func() {
 		It("returns an error if fetching the logs fails", func() {
 			blockChain := fakes.NewMockBlockChain()
 			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
-			fetcher := fetch.NewFetcher(blockChain)
+			logFetcher := fetcher.NewLogFetcher(blockChain)

-			_, err := fetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})
+			_, err := logFetcher.FetchLogs([]common.Address{}, []common.Hash{}, core.Header{})

 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))
@@ -0,0 +1,54 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher
+
+import (
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fs"
+)
+
+type IStorageFetcher interface {
+	FetchStorageDiffs(chan<- utils.StorageDiffRow, chan<- error)
+}
+
+type CsvTailStorageFetcher struct {
+	tailer fs.Tailer
+}
+
+func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher {
+	return CsvTailStorageFetcher{tailer: tailer}
+}
+
+func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+	t, tailErr := storageFetcher.tailer.Tail()
+	if tailErr != nil {
+		errs <- tailErr
+	}
+	log.Debug("fetching storage diffs...")
+	for line := range t.Lines {
+		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+		if parseErr != nil {
+			errs <- parseErr
+		} else {
+			out <- row
+		}
+	}
+}
@@ -0,0 +1,99 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package fetcher_test
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/hpcloud/tail"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+)
+
+var _ = Describe("Csv Tail Storage Fetcher", func() {
+	var (
+		errorsChannel  chan error
+		mockTailer     *fakes.MockTailer
+		rowsChannel    chan utils.StorageDiffRow
+		storageFetcher fetcher.CsvTailStorageFetcher
+	)
+
+	BeforeEach(func() {
+		errorsChannel = make(chan error)
+		rowsChannel = make(chan utils.StorageDiffRow)
+		mockTailer = fakes.NewMockTailer()
+		storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer)
+	})
+
+	It("adds error to errors channel if tailing file fails", func(done Done) {
+		mockTailer.TailErr = fakes.FakeError
+
+		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+
+		Expect(<-errorsChannel).To(MatchError(fakes.FakeError))
+		close(done)
+	})
+
+	It("adds parsed csv row to rows channel for storage diff", func(done Done) {
+		line := getFakeLine()
+
+		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		mockTailer.Lines <- line
+
+		expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(<-rowsChannel).To(Equal(expectedRow))
+		close(done)
+	})
+
+	It("adds error to errors channel if parsing csv fails", func(done Done) {
+		line := &tail.Line{Text: "invalid"}
+
+		go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel)
+		mockTailer.Lines <- line
+
+		Expect(<-errorsChannel).To(HaveOccurred())
+		select {
+		case <-rowsChannel:
+			Fail("value passed to rows channel on error")
+		default:
+			Succeed()
+		}
+		close(done)
+	})
+})
+
+func getFakeLine() *tail.Line {
+	address := common.HexToAddress("0x1234567890abcdef")
+	blockHash := []byte{4, 5, 6}
+	blockHeight := int64(789)
+	storageKey := []byte{9, 8, 7}
+	storageValue := []byte{6, 5, 4}
+	return &tail.Line{
+		Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address.Bytes()), common.Bytes2Hex(blockHash),
+			blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
+		Time: time.Time{},
+		Err:  nil,
+	}
+}
@@ -53,6 +53,6 @@ func (converter *MockConverter) SetToEntityConverterError(err error) {
 	converter.entityConverterError = err
 }

-func (c *MockConverter) SetToModelConverterError(err error) {
-	c.modelConverterError = err
+func (converter *MockConverter) SetToModelConverterError(err error) {
+	converter.modelConverterError = err
 }
@@ -14,10 +14,26 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.

-package event
+package mocks

-import "github.com/ethereum/go-ethereum/core/types"
+import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"

-type LogNoteConverter interface {
-	ToModels(ethLog []types.Log) ([]interface{}, error)
+type MockStorageFetcher struct {
+	RowsToReturn []utils.StorageDiffRow
+	ErrsToReturn []error
+}
+
+func NewMockStorageFetcher() *MockStorageFetcher {
+	return &MockStorageFetcher{}
+}
+
+func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) {
+	defer close(out)
+	defer close(errs)
+	for _, err := range fetcher.ErrsToReturn {
+		errs <- err
+	}
+	for _, row := range fetcher.RowsToReturn {
+		out <- row
+	}
 }
@@ -21,11 +21,26 @@ import (
 )

 type MockStorageQueue struct {
-	AddCalled bool
-	AddError  error
+	AddCalled      bool
+	AddError       error
+	AddPassedRow   utils.StorageDiffRow
+	DeleteErr      error
+	DeletePassedId int
+	GetAllErr      error
+	RowsToReturn   []utils.StorageDiffRow
 }

 func (queue *MockStorageQueue) Add(row utils.StorageDiffRow) error {
 	queue.AddCalled = true
+	queue.AddPassedRow = row
 	return queue.AddError
 }
+
+func (queue *MockStorageQueue) Delete(id int) error {
+	queue.DeletePassedId = id
+	return queue.DeleteErr
+}
+
+func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
+	return queue.RowsToReturn, queue.GetAllErr
+}
@@ -19,7 +19,6 @@ package mocks
 import (
 	"github.com/ethereum/go-ethereum/core/types"

-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -33,7 +32,7 @@ type MockTransformer struct {
 	config           shared_t.EventTransformerConfig
 }

-func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error {
 	if mh.ExecuteError != nil {
 		return mh.ExecuteError
 	}
@@ -69,31 +69,6 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
 	return result, err
 }

-func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-
-	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
-				AND headers.block_number >= $1
-				AND headers.eth_node_fingerprint = $2`
-		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
-	} else {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
-				AND headers.block_number >= $1
-				AND headers.block_number <= $2
-				AND headers.eth_node_fingerprint = $3`
-		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
-	}
-
-	return result, err
-}
-
 func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
 	// Query returns `[]driver.Value`, nullable polymorphic interface
 	var queryResult []driver.Value
@@ -101,7 +76,8 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
 		`SELECT column_name FROM information_schema.columns
 		WHERE table_schema = 'public'
 			AND table_name = 'checked_headers'
-			AND column_name ~ '_checked';`
+			AND column_name <> 'id'
+			AND column_name <> 'header_id';`

 	err := db.Select(&queryResult, columnNamesQuery)
 	if err != nil {
@@ -121,37 +97,47 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
 	return columnNames, nil
 }

-// Builds a SQL string that checks if any column value is 0, given the column names.
+// Builds a SQL string that checks if any column should be checked/rechecked.
 // Defaults to FALSE when no columns are provided.
 // Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
 //     [] => "FALSE"
-func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
-
-	var result bytes.Buffer
-
+func CreateHeaderCheckedPredicateSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
 	if len(boolColumns) == 0 {
 		return "FALSE"
 	}

-	result.WriteString("NOT (")
+	if recheckHeaders {
+		return createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns)
+	} else {
+		return createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns)
+	}
+}
+
+func createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns []string) string {
+	var result bytes.Buffer
+	result.WriteString(" (")

 	// Loop excluding last column name
 	for _, column := range boolColumns[:len(boolColumns)-1] {
-
-		if recheckHeaders {
-			result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
-		} else {
-			result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
-		}
+		result.WriteString(fmt.Sprintf("%v=0 OR ", column))
 	}

-	// No trailing "OR" for the last column name
-	if recheckHeaders {
-		result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
-	} else {
-		result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
+	result.WriteString(fmt.Sprintf("%v=0)", boolColumns[len(boolColumns)-1]))
+
+	return result.String()
+}
+
+func createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns []string) string {
+	var result bytes.Buffer
+	result.WriteString(" (")

+	// Loop excluding last column name
+	for _, column := range boolColumns[:len(boolColumns)-1] {
+		result.WriteString(fmt.Sprintf("%v<%s OR ", column, constants.RecheckHeaderCap))
 	}
+
+	// No trailing "OR" for the last column name
+	result.WriteString(fmt.Sprintf("%v<%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))

 	return result.String()
 }
@@ -26,7 +26,7 @@ import (

 	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	shared "github.com/vulcanize/vulcanizedb/libraries/shared/repository"
-	r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	r2 "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -36,12 +36,12 @@ import (
 )

 var _ = Describe("Repository", func() {
-	Describe("MarkHeaderChecked", func() {
-		var (
-			checkedHeadersColumn string
-			db                   *postgres.DB
-		)
+	var (
+		checkedHeadersColumn string
+		db                   *postgres.DB
+	)

+	Describe("MarkHeaderChecked", func() {
 		BeforeEach(func() {
 			db = test_config.NewTestDB(test_config.NewTestNode())
 			test_config.CleanTestDB(db)
@@ -73,11 +73,6 @@ var _ = Describe("Repository", func() {
 	})

 	Describe("MarkHeaderCheckedInTransaction", func() {
-		var (
-			checkedHeadersColumn string
-			db                   *postgres.DB
-		)
-
 		BeforeEach(func() {
 			db = test_config.NewTestDB(test_config.NewTestNode())
 			test_config.CleanTestDB(db)
@@ -114,16 +109,17 @@ var _ = Describe("Repository", func() {

 	Describe("MissingHeaders", func() {
 		var (
-			db                       *postgres.DB
 			headerRepository         datastore.HeaderRepository
 			startingBlockNumber      int64
 			endingBlockNumber        int64
 			eventSpecificBlockNumber int64
+			outOfRangeBlockNumber    int64
 			blockNumbers             []int64
 			headerIDs                []int64
 			notCheckedSQL            string
 			err                      error
 			hr                       r2.HeaderRepository
+			columnNames              []string
 		)

 		BeforeEach(func() {
@@ -133,14 +129,14 @@ var _ = Describe("Repository", func() {
 			hr = r2.NewHeaderRepository(db)
 			hr.AddCheckColumns(getExpectedColumnNames())

-			columnNames, err := shared.GetCheckedColumnNames(db)
+			columnNames, err = shared.GetCheckedColumnNames(db)
 			Expect(err).NotTo(HaveOccurred())
-			notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
+			notCheckedSQL = shared.CreateHeaderCheckedPredicateSQL(columnNames, constants.HeaderMissing)

 			startingBlockNumber = rand.Int63()
 			eventSpecificBlockNumber = startingBlockNumber + 1
 			endingBlockNumber = startingBlockNumber + 2
-			outOfRangeBlockNumber := endingBlockNumber + 1
+			outOfRangeBlockNumber = endingBlockNumber + 1

 			blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}

@@ -157,6 +153,7 @@ var _ = Describe("Repository", func() {
 		})

 		It("only treats headers as checked if the event specific logs have been checked", func() {
+			//add a checked_header record, but don't mark it check for any of the columns
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())

@@ -191,83 +188,47 @@ var _ = Describe("Repository", func() {
 			Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
 			Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
 		})
-	})

-	Describe("RecheckHeaders", func() {
-		var (
-			checkedHeadersColumn                                      string
-			db                                                        *postgres.DB
-			headerOneID, headerTwoID, headerThreeID, headerFourID     int64
-			headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
-		)
+		It("handles an ending block of -1 ", func() {
+			endingBlock := int64(-1)
+			headers, err := shared.MissingHeaders(startingBlockNumber, endingBlock, db, notCheckedSQL)

-		BeforeEach(func() {
-			db = test_config.NewTestDB(test_config.NewTestNode())
-			test_config.CleanTestDB(db)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(4))
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
+			Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))

-			// create header checked column
-			checkedHeadersColumn = "test_column_checked"
-			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
-			Expect(migrateErr).NotTo(HaveOccurred())
+		})

-			// create headers
-			headerRepository := repositories.NewHeaderRepository(db)
-			headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
-			Expect(headerOneErr).NotTo(HaveOccurred())
-			headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
-			Expect(headerTwoErr).NotTo(HaveOccurred())
-			headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
-			Expect(headerThreeErr).NotTo(HaveOccurred())
-			headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
-			Expect(headerFourErr).NotTo(HaveOccurred())
+		It("when a the `notCheckedSQL` argument allows for rechecks it returns headers where the checked count is less than the maximum", func() {
+			columnName := columnNames[0]
+			recheckedSQL := shared.CreateHeaderCheckedPredicateSQL([]string{columnName}, constants.HeaderRecheck)
+			// mark every header checked at least once
+			// header 4 is marked the maximum number of times, it it is not longer checked

-			// mark every header checked at least once, with one fully rechecked (headerThree)
 			maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
 			Expect(intConversionErr).NotTo(HaveOccurred())
-			_, markHeaderOneCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerOneID, maxCheckCount)
-			Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderTwoCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerTwoID, maxCheckCount)
-			Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderThreeCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerThreeID, maxCheckCount+1)
-			Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderFourCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerFourID, maxCheckCount)
-			Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
-		})

-		AfterEach(func() {
-			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
-			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
-		})
+			markHeaderOneErr := shared.MarkHeaderChecked(headerIDs[0], db, columnName)
+			Expect(markHeaderOneErr).NotTo(HaveOccurred())
+			markHeaderTwoErr := shared.MarkHeaderChecked(headerIDs[1], db, columnName)
+			Expect(markHeaderTwoErr).NotTo(HaveOccurred())
+			markHeaderThreeErr := shared.MarkHeaderChecked(headerIDs[2], db, columnName)
+			Expect(markHeaderThreeErr).NotTo(HaveOccurred())
+			for i := 0; i <= maxCheckCount; i++ {
+				markHeaderFourErr := shared.MarkHeaderChecked(headerIDs[3], db, columnName)
+				Expect(markHeaderFourErr).NotTo(HaveOccurred())
+			}

-		Describe("when no ending block number (ending block number == -1)", func() {
-			It("returns all headers since starting block where checked count is less than cap", func() {
-				headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
+			headers, err := shared.MissingHeaders(1, -1, db, recheckedSQL)

-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(headers)).To(Equal(3))
-				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
-				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
-				Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
-			})
-		})
-
-		Describe("when ending block number specified", func() {
-			It("returns headers between starting and ending block where checked count is less than cap", func() {
-				headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(headers)).To(Equal(2))
-				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
-				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
-			})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(headers)).To(Equal(3))
+			Expect(headers[0].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
+			Expect(headers[1].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
+			Expect(headers[2].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
 		})
 	})

@@ -285,41 +246,58 @@ var _ = Describe("Repository", func() {
 		})
 	})

-	Describe("CreateNotCheckedSQL", func() {
-		It("generates a correct SQL string for one column", func() {
-			columns := []string{"columnA"}
-			expected := "NOT (columnA!=0)"
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
-			Expect(actual).To(Equal(expected))
+	Describe("CreateHeaderCheckedPredicateSQL", func() {
+		Describe("for headers that haven't been checked for logs", func() {
+			It("generates a correct SQL string for one column", func() {
+				columns := []string{"columnA"}
+				expected := " (columnA=0)"
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
+				Expect(actual).To(Equal(expected))
+			})
+
+			It("generates a correct SQL string for several columns", func() {
+				columns := []string{"columnA", "columnB"}
+				expected := " (columnA=0 OR columnB=0)"
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
+				Expect(actual).To(Equal(expected))
+			})
+
+			It("defaults to FALSE when there are no columns", func() {
+				expected := "FALSE"
+				actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderMissing)
+				Expect(actual).To(Equal(expected))
+			})
 		})

-		It("generates a correct SQL string for several columns", func() {
-			columns := []string{"columnA", "columnB"}
-			expected := "NOT (columnA!=0 AND columnB!=0)"
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
-			Expect(actual).To(Equal(expected))
-		})
+		Describe("for headers that are being rechecked for logs", func() {
+			It("generates a correct SQL string for rechecking headers for one column", func() {
+				columns := []string{"columnA"}
+				expected := fmt.Sprintf(" (columnA<%s)", constants.RecheckHeaderCap)
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
+				Expect(actual).To(Equal(expected))
+			})

-		It("defaults to FALSE when there are no columns", func() {
-			expected := "FALSE"
-			actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
-			Expect(actual).To(Equal(expected))
-		})
+			It("generates a correct SQL string for rechecking headers for several columns", func() {
+				columns := []string{"columnA", "columnB"}
+				expected := fmt.Sprintf(" (columnA<%s OR columnB<%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
+				Expect(actual).To(Equal(expected))
+			})

-		It("generates a correct SQL string for rechecking headers", func() {
-			columns := []string{"columnA", "columnB"}
-			expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
-			Expect(actual).To(Equal(expected))
+			It("defaults to FALSE when there are no columns", func() {
+				expected := "FALSE"
+				actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderRecheck)
+				Expect(actual).To(Equal(expected))
+			})
 		})
 	})
 })

 func getExpectedColumnNames() []string {
 	return []string{
-		"column_1_checked",
-		"column_2_checked",
-		"column_3_checked",
-		"column_4_checked",
+		"column_1",
+		"column_2",
+		"column_3",
+		"column_4",
 	}
 }
@@ -47,7 +47,7 @@ const (
 )

 func GetMapping(indexOnContract, key string) common.Hash {
-	keyBytes := common.FromHex("0x" + key + indexOnContract)
+	keyBytes := common.FromHex(key + indexOnContract)
 	encoded := crypto.Keccak256(keyBytes)
 	return common.BytesToHash(encoded)
 }
@@ -21,6 +21,16 @@ var _ = Describe("Mappings", func() {
 			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
 			Expect(storageKey).To(Equal(expectedStorageKey))
 		})
+
+		It("returns same result if value includes hex prefix", func() {
+			indexOfMappingOnContract := storage.IndexZero
+			keyForDesiredValueInMapping := "0x1234567890abcdef"
+
+			storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+
+			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
 	})

 	Describe("GetNestedMapping", func() {
@@ -23,6 +23,8 @@ import (

 type IStorageQueue interface {
 	Add(row utils.StorageDiffRow) error
+	Delete(id int) error
+	GetAll() ([]utils.StorageDiffRow, error)
 }

 type StorageQueue struct {
@@ -36,7 +38,18 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
 func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
 	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
 		block_hash, block_height, storage_key, storage_value) VALUES
-		($1, $2, $3, $4, $5)`, row.Contract.Bytes(), row.BlockHash.Bytes(),
+		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, row.Contract.Bytes(), row.BlockHash.Bytes(),
 		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
 	return err
 }
+
+func (queue StorageQueue) Delete(id int) error {
+	_, err := queue.db.Exec(`DELETE FROM public.queued_storage WHERE id = $1`, id)
+	return err
+}
+
+func (queue StorageQueue) GetAll() ([]utils.StorageDiffRow, error) {
+	var result []utils.StorageDiffRow
+	err := queue.db.Select(&result, `SELECT * FROM public.queued_storage`)
+	return result, err
+}
@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package storage_test

 import (
@@ -7,27 +23,90 @@ import (

 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )

 var _ = Describe("Storage queue", func() {
-	It("adds a storage row to the db", func() {
-		row := utils.StorageDiffRow{
+	var (
+		db    *postgres.DB
+		row   utils.StorageDiffRow
+		queue storage.IStorageQueue
+	)
+
+	BeforeEach(func() {
+		row = utils.StorageDiffRow{
 			Contract:     common.HexToAddress("0x123456"),
 			BlockHash:    common.HexToHash("0x678901"),
 			BlockHeight:  987,
 			StorageKey:   common.HexToHash("0x654321"),
 			StorageValue: common.HexToHash("0x198765"),
 		}
-		db := test_config.NewTestDB(test_config.NewTestNode())
-		queue := storage.NewStorageQueue(db)
-
+		db = test_config.NewTestDB(test_config.NewTestNode())
+		test_config.CleanTestDB(db)
+		queue = storage.NewStorageQueue(db)
 		addErr := queue.Add(row)
-
 		Expect(addErr).NotTo(HaveOccurred())
-		var result utils.StorageDiffRow
-		getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
+	})
+
+	Describe("Add", func() {
+		It("adds a storage row to the db", func() {
+			var result utils.StorageDiffRow
+			getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(result).To(Equal(row))
+		})
+
+		It("does not duplicate storage rows", func() {
+			addErr := queue.Add(row)
+			Expect(addErr).NotTo(HaveOccurred())
+			var count int
+			getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(count).To(Equal(1))
+		})
+	})
+
+	It("deletes storage row from db", func() {
+		rows, getErr := queue.GetAll()
 		Expect(getErr).NotTo(HaveOccurred())
-		Expect(result).To(Equal(row))
+		Expect(len(rows)).To(Equal(1))
+
+		err := queue.Delete(rows[0].Id)
+
+		Expect(err).NotTo(HaveOccurred())
+		remainingRows, secondGetErr := queue.GetAll()
+		Expect(secondGetErr).NotTo(HaveOccurred())
+		Expect(len(remainingRows)).To(BeZero())
+	})
+
+	It("gets all storage rows from db", func() {
+		rowTwo := utils.StorageDiffRow{
+			Contract:     common.HexToAddress("0x123456"),
+			BlockHash:    common.HexToHash("0x678902"),
+			BlockHeight:  988,
+			StorageKey:   common.HexToHash("0x654322"),
+			StorageValue: common.HexToHash("0x198766"),
+		}
+		addErr := queue.Add(rowTwo)
+		Expect(addErr).NotTo(HaveOccurred())
+
+		rows, err := queue.GetAll()
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(rows)).To(Equal(2))
+		Expect(rows[0]).NotTo(Equal(rows[1]))
+		Expect(rows[0].Id).NotTo(BeZero())
+		Expect(rows[0].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
+		Expect(rows[0].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
+		Expect(rows[0].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
+		Expect(rows[0].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
+		Expect(rows[0].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
+		Expect(rows[1].Id).NotTo(BeZero())
+		Expect(rows[1].Contract).To(Or(Equal(row.Contract), Equal(rowTwo.Contract)))
+		Expect(rows[1].BlockHash).To(Or(Equal(row.BlockHash), Equal(rowTwo.BlockHash)))
+		Expect(rows[1].BlockHeight).To(Or(Equal(row.BlockHeight), Equal(rowTwo.BlockHeight)))
+		Expect(rows[1].StorageKey).To(Or(Equal(row.StorageKey), Equal(rowTwo.StorageKey)))
+		Expect(rows[1].StorageValue).To(Or(Equal(row.StorageValue), Equal(rowTwo.StorageValue)))
 	})
 })
@@ -23,27 +23,30 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )

+const (
+	bitsPerByte = 8
+)
+
 func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
 	switch metadata.Type {
 	case Uint256:
-		return decodeUint256(row.StorageValue.Bytes()), nil
+		return decodeInteger(row.StorageValue.Bytes()), nil
 	case Uint48:
-		return decodeUint48(row.StorageValue.Bytes()), nil
+		return decodeInteger(row.StorageValue.Bytes()), nil
+	case Uint128:
+		return decodeInteger(row.StorageValue.Bytes()), nil
 	case Address:
 		return decodeAddress(row.StorageValue.Bytes()), nil
 	case Bytes32:
 		return row.StorageValue.Hex(), nil
+	case PackedSlot:
+		return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
 	default:
 		panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
 	}
 }

-func decodeUint256(raw []byte) string {
-	n := big.NewInt(0).SetBytes(raw)
-	return n.String()
-}
-
-func decodeUint48(raw []byte) string {
+func decodeInteger(raw []byte) string {
 	n := big.NewInt(0).SetBytes(raw)
 	return n.String()
 }
@@ -51,3 +54,53 @@ func decodeUint48(raw []byte) string {
 func decodeAddress(raw []byte) string {
 	return common.BytesToAddress(raw).Hex()
 }
+
+func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string {
+	storageSlotData := raw
+	decodedStorageSlotItems := map[int]string{}
+	numberOfTypes := len(packedTypes)
+
+	for position := 0; position < numberOfTypes; position++ {
+		//get length of remaining storage date
+		lengthOfStorageData := len(storageSlotData)
+
+		//get item details (type, length, starting index, value bytes)
+		itemType := packedTypes[position]
+		lengthOfItem := getNumberOfBytes(itemType)
+		itemStartingIndex := lengthOfStorageData - lengthOfItem
+		itemValueBytes := storageSlotData[itemStartingIndex:]
+
+		//decode item's bytes and set in results map
+		decodedValue := decodeIndividualItem(itemValueBytes, itemType)
+		decodedStorageSlotItems[position] = decodedValue
+
+		//pop last item off raw slot data before moving on
+		storageSlotData = storageSlotData[0:itemStartingIndex]
+	}
+
+	return decodedStorageSlotItems
+}
+
+func decodeIndividualItem(itemBytes []byte, valueType ValueType) string {
+	switch valueType {
+	case Uint48, Uint128:
+		return decodeInteger(itemBytes)
+	case Address:
+		return decodeAddress(itemBytes)
+	default:
+		panic(fmt.Sprintf("can't decode unknown type: %d", valueType))
+	}
+}
+
+func getNumberOfBytes(valueType ValueType) int {
+	switch valueType {
+	case Uint48:
+		return 48 / bitsPerByte
+	case Uint128:
+		return 128 / bitsPerByte
+	case Address:
+		return 20
+	default:
+		panic(fmt.Sprintf("ValueType %d not recognized", valueType))
+	}
+}
@@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
 	})

+	It("decodes uint128", func() {
+		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
+		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		metadata := utils.StorageValueMetadata{Type: utils.Uint128}
+
+		result, err := utils.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
+	})
+
 	It("decodes uint48", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
 		row := utils.StorageDiffRow{StorageValue: fakeInt}
@@ -59,4 +70,105 @@ var _ = Describe("Storage decoder", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(fakeAddress.Hex()))
 	})
+
+	Describe("when there are multiple items packed in the storage slot", func() {
+		It("decodes uint48 items", func() {
+			//this is a real storage data example
+			packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint48
+			packedTypes[1] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+		})
+
+		It("decodes 5 uint48 items", func() {
+			//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
+			//real storage data that has several items packed into it
+			packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
+
+			packedStorage := common.HexToHash(packedStorageHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint48
+			packedTypes[1] = utils.Uint48
+			packedTypes[2] = utils.Uint48
+			packedTypes[3] = utils.Uint48
+			packedTypes[4] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+			Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String()))
+			Expect(decodedValues[3]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String()))
+			Expect(decodedValues[4]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String()))
+		})
+
+		It("decodes 2 uint128 items", func() {
+			//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
+			//real storage data that has several items packed into it
+			packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
+				"0000000000000000AB54A98CEB1F0AD2"
+			packedStorage := common.HexToHash(packedStorageHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint128
+			packedTypes[1] = utils.Uint128
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("38D7EA4C67FF8E502B6730000").Bytes()).String()))
+		})
+
+		It("decodes address + 2 uint48s", func() {
+			//TODO: replace with real data when available
+			addressHex := "0000000000000000000000000000000000012345"
+			packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Address
+			packedTypes[1] = utils.Uint48
+			packedTypes[2] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal("0x" + addressHex))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+		})
+	})
 })
@@ -25,6 +25,7 @@ import (
 const ExpectedRowLength = 5

 type StorageDiffRow struct {
+	Id           int
 	Contract     common.Address
 	BlockHash    common.Hash `db:"block_hash"`
 	BlockHeight  int         `db:"block_height"`
@@ -16,27 +16,54 @@

 package utils

+import "fmt"
+
 type ValueType int

 const (
 	Uint256 ValueType = iota
 	Uint48
+	Uint128
 	Bytes32
 	Address
+	PackedSlot
 )

 type Key string

 type StorageValueMetadata struct {
-	Name string
-	Keys map[Key]string
-	Type ValueType
+	Name        string
+	Keys        map[Key]string
+	Type        ValueType
+	PackedNames map[int]string    //zero indexed position in map => name of packed item
+	PackedTypes map[int]ValueType //zero indexed position in map => type of packed item
 }

-func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata {
+func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata {
+	return getMetadata(name, keys, valueType, nil, nil)
+}
+
+func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
+	return getMetadata(name, keys, valueType, packedNames, packedTypes)
+}
+
+func getMetadata(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
+	assertPackedSlotArgs(valueType, packedNames, packedTypes)
+
 	return StorageValueMetadata{
-		Name: name,
-		Keys: keys,
-		Type: t,
+		Name:        name,
+		Keys:        keys,
+		Type:        valueType,
+		PackedNames: packedNames,
+		PackedTypes: packedTypes,
 	}
 }
+
+func assertPackedSlotArgs(valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) {
+	if valueType == PackedSlot && (packedTypes == nil || packedNames == nil) {
+		panic(fmt.Sprintf("ValueType is PackedSlot. Expected PackedNames and PackedTypes to not be nil, but got PackedNames = %v and PackedTypes = %v", packedNames, packedTypes))
+	} else if (packedNames != nil && packedTypes != nil) && valueType != PackedSlot {
+		panic(fmt.Sprintf("PackedNames and PackedTypes passed in. Expected ValueType to equal PackedSlot (%v), but got %v.", PackedSlot, valueType))
+	}
+
+}
@@ -7,7 +7,7 @@ import (
 )

 var _ = Describe("Storage value metadata getter", func() {
-	It("returns a storage value metadata instance with corresponding fields assigned", func() {
+	It("returns storage value metadata for a single storage variable", func() {
 		metadataName := "fake_name"
 		metadataKeys := map[utils.Key]string{"key": "value"}
 		metadataType := utils.Uint256
@@ -19,4 +19,60 @@ var _ = Describe("Storage value metadata getter", func() {
 		}
 		Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
 	})
+
+	Describe("metadata for a packed storaged slot", func() {
+		It("returns metadata for multiple storage variables", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedNames := map[int]string{0: "name"}
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			expectedMetadata := utils.StorageValueMetadata{
+				Name:        metadataName,
+				Keys:        metadataKeys,
+				Type:        metadataType,
+				PackedTypes: metadataPackedTypes,
+				PackedNames: metadataPackedNames,
+			}
+			Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata))
+		})
+
+		It("panics if PackedTypes are nil when the type is PackedSlot", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedNames := map[int]string{0: "name"}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, nil)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+
+		It("panics if PackedNames are nil when the type is PackedSlot", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, nil, metadataPackedTypes)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+
+		It("panics if valueType is not PackedSlot if PackedNames is populated", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.Uint48
+			metadataPackedNames := map[int]string{0: "name"}
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+	})
 })
@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package transactions

 import (
@@ -28,6 +44,9 @@ func NewTransactionsSyncer(db *postgres.DB, blockChain core.BlockChain) Transact

 func (syncer TransactionsSyncer) SyncTransactions(headerID int64, logs []types.Log) error {
 	transactionHashes := getUniqueTransactionHashes(logs)
+	if len(transactionHashes) < 1 {
+		return nil
+	}
 	transactions, transactionErr := syncer.BlockChain.GetTransactions(transactionHashes)
 	if transactionErr != nil {
 		return transactionErr
@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package transactions_test

 import (
@@ -24,12 +40,19 @@ var _ = Describe("Transaction syncer", func() {
 	})

 	It("fetches transactions for logs", func() {
-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})

 		Expect(err).NotTo(HaveOccurred())
 		Expect(blockChain.GetTransactionsCalled).To(BeTrue())
 	})

+	It("does not fetch transactions if no logs", func() {
+		err := syncer.SyncTransactions(0, []types.Log{})
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(blockChain.GetTransactionsCalled).To(BeFalse())
+	})
+
 	It("only fetches transactions with unique hashes", func() {
 		err := syncer.SyncTransactions(0, []types.Log{{
 			TxHash: fakes.FakeHash,
@@ -44,7 +67,7 @@ var _ = Describe("Transaction syncer", func() {
 	It("returns error if fetching transactions fails", func() {
 		blockChain.GetTransactionsError = fakes.FakeError

-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -55,7 +78,7 @@ var _ = Describe("Transaction syncer", func() {
 		mockHeaderRepository := fakes.NewMockHeaderRepository()
 		syncer.Repository = mockHeaderRepository

-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})

 		Expect(err).NotTo(HaveOccurred())
 		Expect(mockHeaderRepository.CreateTransactionsCalled).To(BeTrue())
@@ -67,7 +90,7 @@ var _ = Describe("Transaction syncer", func() {
 		mockHeaderRepository.CreateTransactionsError = fakes.FakeError
 		syncer.Repository = mockHeaderRepository

-		err := syncer.SyncTransactions(0, []types.Log{})
+		err := syncer.SyncTransactions(0, []types.Log{{TxHash: fakes.FakeHash}})

 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package transactions_test

 import (
@@ -20,13 +20,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"

-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )

 type EventTransformer interface {
-	Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
+	Execute(logs []types.Log, header core.Header) error
 	GetConfig() EventTransformerConfig
 }

@@ -54,10 +54,10 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error {
 		watcher.Transformers = append(watcher.Transformers, t)
 	}

-	for _, transformer := range watcher.Transformers {
-		err := transformer.Init()
+	for _, contractTransformer := range watcher.Transformers {
+		err := contractTransformer.Init()
 		if err != nil {
-			log.Print("Unable to initialize transformer:", transformer.GetConfig().Name, err)
+			log.Print("Unable to initialize transformer:", contractTransformer.GetConfig().Name, err)
 			return err
 		}
 	}
@@ -65,10 +65,10 @@ func (watcher *ContractWatcher) AddTransformers(inits interface{}) error {
 }

 func (watcher *ContractWatcher) Execute() error {
-	for _, transformer := range watcher.Transformers {
-		err := transformer.Execute()
+	for _, contractTransformer := range watcher.Transformers {
+		err := contractTransformer.Execute()
 		if err != nil {
-			log.Error("Unable to execute transformer:", transformer.GetConfig().Name, err)
+			log.Error("Unable to execute transformer:", contractTransformer.GetConfig().Name, err)
 			return err
 		}
 	}
@@ -37,7 +37,7 @@ type EventWatcher struct {
 	Transformers  []transformer.EventTransformer
 	BlockChain    core.BlockChain
 	DB            *postgres.DB
-	Fetcher       fetcher.LogFetcher
+	Fetcher       fetcher.ILogFetcher
 	Chunker       chunker.Chunker
 	Addresses     []common.Address
 	Topics        []common.Hash
@@ -47,7 +47,7 @@ type EventWatcher struct {

 func NewEventWatcher(db *postgres.DB, bc core.BlockChain) EventWatcher {
 	logChunker := chunker.NewLogChunker()
-	logFetcher := fetcher.NewFetcher(bc)
+	logFetcher := fetcher.NewLogFetcher(bc)
 	transactionSyncer := transactions.NewTransactionsSyncer(db, bc)
 	return EventWatcher{
 		BlockChain: bc,
@@ -96,11 +96,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
 	if err != nil {
 		return err
 	}
-	notCheckedSQL := repository.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
+	notCheckedSQL := repository.CreateHeaderCheckedPredicateSQL(checkedColumnNames, recheckHeaders)

 	missingHeaders, err := repository.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
 	if err != nil {
-		logrus.Error("Fetching of missing headers failed in watcher!")
+		logrus.Error("Couldn't fetch missing headers in watcher: ", err)
 		return err
 	}

@@ -108,7 +108,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
 		// TODO Extend FetchLogs for doing several blocks at a time
 		logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
 		if err != nil {
-			logrus.Errorf("Error while fetching logs for header %v in watcher", header.Id)
+			logrus.WithFields(logrus.Fields{
+				"headerId":    header.Id,
+				"headerHash":  header.Hash,
+				"blockNumber": header.BlockNumber,
+			}).Errorf("Couldn't fetch logs for header: %v", err)
 			return err
 		}

@@ -120,6 +124,7 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti

 		transformErr := watcher.transformLogs(logs, header)
 		if transformErr != nil {
+			logrus.Error("Could not transform logs: ", transformErr)
 			return transformErr
 		}
 	}
@@ -134,7 +139,7 @@ func (watcher *EventWatcher) transformLogs(logs []types.Log, header core.Header)
 	for _, t := range watcher.Transformers {
 		transformerName := t.GetConfig().TransformerName
 		logChunk := chunkedLogs[transformerName]
-		err := t.Execute(logChunk, header, constants.HeaderMissing)
+		err := t.Execute(logChunk, header)
 		if err != nil {
 			logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
 			return err
@@ -17,73 +17,100 @@
 package watcher

 import (
+	"fmt"
 	"reflect"
-	"strings"
+	"time"

 	"github.com/ethereum/go-ethereum/common"
 	"github.com/sirupsen/logrus"

+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/fs"
 )

 type StorageWatcher struct {
-	db           *postgres.DB
-	tailer       fs.Tailer
-	Queue        storage.IStorageQueue
-	Transformers map[common.Address]transformer.StorageTransformer
+	db             *postgres.DB
+	StorageFetcher fetcher.IStorageFetcher
+	Queue          storage.IStorageQueue
+	Transformers   map[common.Address]transformer.StorageTransformer
 }

-func NewStorageWatcher(tailer fs.Tailer, db *postgres.DB) StorageWatcher {
+func NewStorageWatcher(fetcher fetcher.IStorageFetcher, db *postgres.DB) StorageWatcher {
 	transformers := make(map[common.Address]transformer.StorageTransformer)
 	queue := storage.NewStorageQueue(db)
 	return StorageWatcher{
-		db:           db,
-		tailer:       tailer,
-		Queue:        queue,
-		Transformers: transformers,
+		db:             db,
+		StorageFetcher: fetcher,
+		Queue:          queue,
+		Transformers:   transformers,
 	}
 }

-func (watcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
+func (storageWatcher StorageWatcher) AddTransformers(initializers []transformer.StorageTransformerInitializer) {
 	for _, initializer := range initializers {
-		storageTransformer := initializer(watcher.db)
-		watcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
+		storageTransformer := initializer(storageWatcher.db)
+		storageWatcher.Transformers[storageTransformer.ContractAddress()] = storageTransformer
 	}
 }

-func (watcher StorageWatcher) Execute() error {
-	t, tailErr := watcher.tailer.Tail()
-	if tailErr != nil {
-		return tailErr
-	}
-	for line := range t.Lines {
-		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
-		if parseErr != nil {
-			return parseErr
+func (storageWatcher StorageWatcher) Execute(rows chan utils.StorageDiffRow, errs chan error, queueRecheckInterval time.Duration) {
+	ticker := time.NewTicker(queueRecheckInterval)
+	go storageWatcher.StorageFetcher.FetchStorageDiffs(rows, errs)
+	for {
+		select {
+		case fetchErr := <-errs:
+			logrus.Warn(fmt.Sprintf("error fetching storage diffs: %s", fetchErr))
+		case row := <-rows:
+			storageWatcher.processRow(row)
+		case <-ticker.C:
+			storageWatcher.processQueue()
 		}
-		storageTransformer, ok := watcher.Transformers[row.Contract]
+	}
+}
+
+func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
+	storageTransformer, ok := storageWatcher.Transformers[row.Contract]
+	if !ok {
+		// ignore rows from unwatched contracts
+		return
+	}
+	executeErr := storageTransformer.Execute(row)
+	if executeErr != nil {
+		logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
+		queueErr := storageWatcher.Queue.Add(row)
+		if queueErr != nil {
+			logrus.Warn(fmt.Sprintf("error queueing storage diff: %s", queueErr))
+		}
+	}
+}
+
+func (storageWatcher StorageWatcher) processQueue() {
+	rows, fetchErr := storageWatcher.Queue.GetAll()
+	if fetchErr != nil {
+		logrus.Warn(fmt.Sprintf("error getting queued storage: %s", fetchErr))
+	}
+	for _, row := range rows {
+		storageTransformer, ok := storageWatcher.Transformers[row.Contract]
 		if !ok {
-			logrus.Warn(utils.ErrContractNotFound{Contract: row.Contract.Hex()}.Error())
+			// delete row from queue if address no longer watched
+			storageWatcher.deleteRow(row.Id)
 			continue
 		}
 		executeErr := storageTransformer.Execute(row)
-		if executeErr != nil {
-			if isKeyNotFound(executeErr) {
-				queueErr := watcher.Queue.Add(row)
-				if queueErr != nil {
-					logrus.Warn(queueErr.Error())
-				}
-			} else {
-				logrus.Warn(executeErr.Error())
-			}
-			continue
+		if executeErr == nil {
+			storageWatcher.deleteRow(row.Id)
 		}
 	}
-	return nil
+}
+
+func (storageWatcher StorageWatcher) deleteRow(id int) {
+	deleteErr := storageWatcher.Queue.Delete(id)
+	if deleteErr != nil {
+		logrus.Warn(fmt.Sprintf("error deleting persisted row from queue: %s", deleteErr))
+	}
 }

 func isKeyNotFound(executeErr error) bool {
@@ -17,15 +17,11 @@
 package watcher_test

 import (
-	"errors"
-	"fmt"
 	"io/ioutil"
 	"os"
-	"strings"
 	"time"

 	"github.com/ethereum/go-ethereum/common"
-	"github.com/hpcloud/tail"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
@@ -34,7 +30,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/watcher"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/test_config"
 )
@@ -43,170 +38,207 @@ var _ = Describe("Storage Watcher", func() {
 	It("adds transformers", func() {
 		fakeAddress := common.HexToAddress("0x12345")
 		fakeTransformer := &mocks.MockStorageTransformer{Address: fakeAddress}
-		w := watcher.NewStorageWatcher(&fakes.MockTailer{}, test_config.NewTestDB(core.Node{}))
+		w := watcher.NewStorageWatcher(mocks.NewMockStorageFetcher(), test_config.NewTestDB(test_config.NewTestNode()))

 		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})

 		Expect(w.Transformers[fakeAddress]).To(Equal(fakeTransformer))
 	})

-	It("reads the tail of the storage diffs file", func() {
-		mockTailer := fakes.NewMockTailer()
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
+	Describe("executing watcher", func() {
+		var (
+			errs            chan error
+			mockFetcher     *mocks.MockStorageFetcher
+			mockQueue       *mocks.MockStorageQueue
+			mockTransformer *mocks.MockStorageTransformer
+			row             utils.StorageDiffRow
+			rows            chan utils.StorageDiffRow
+			storageWatcher  watcher.StorageWatcher
+		)

-		assert(func(err error) {
-			Expect(err).To(BeNil())
-			Expect(mockTailer.TailCalled).To(BeTrue())
-		}, w, mockTailer, []*tail.Line{})
-	})
-
-	It("returns error if row parsing fails", func() {
-		mockTailer := fakes.NewMockTailer()
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-		line := &tail.Line{Text: "oops"}
-
-		assert(func(err error) {
-			Expect(err).To(HaveOccurred())
-			Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
-		}, w, mockTailer, []*tail.Line{line})
-	})
-
-	It("logs error if no transformer can parse storage row", func() {
-		mockTailer := fakes.NewMockTailer()
-		address := common.HexToAddress("0x12345")
-		line := getFakeLine(address.Bytes())
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-		tempFile, err := ioutil.TempFile("", "log")
-		defer os.Remove(tempFile.Name())
-		Expect(err).NotTo(HaveOccurred())
-		logrus.SetOutput(tempFile)
-
-		assert(func(err error) {
-			Expect(err).NotTo(HaveOccurred())
-			logContent, readErr := ioutil.ReadFile(tempFile.Name())
-			Expect(readErr).NotTo(HaveOccurred())
-			Expect(string(logContent)).To(ContainSubstring(utils.ErrContractNotFound{Contract: address.Hex()}.Error()))
-		}, w, mockTailer, []*tail.Line{line})
-	})
-
-	It("executes transformer with storage row", func() {
-		address := []byte{1, 2, 3}
-		line := getFakeLine(address)
-		mockTailer := fakes.NewMockTailer()
-		w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-		fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address)}
-		w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-
-		assert(func(err error) {
-			Expect(err).To(BeNil())
-			expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
-			Expect(err).NotTo(HaveOccurred())
-			Expect(fakeTransformer.PassedRow).To(Equal(expectedRow))
-		}, w, mockTailer, []*tail.Line{line})
-	})
-
-	Describe("when executing transformer fails", func() {
-		It("queues row when error is storage key not found", func() {
-			address := []byte{1, 2, 3}
-			line := getFakeLine(address)
-			mockTailer := fakes.NewMockTailer()
-			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-			mockQueue := &mocks.MockStorageQueue{}
-			w.Queue = mockQueue
-			keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
-			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
-			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-
-			assert(func(err error) {
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockQueue.AddCalled).To(BeTrue())
-			}, w, mockTailer, []*tail.Line{line})
+		BeforeEach(func() {
+			errs = make(chan error)
+			rows = make(chan utils.StorageDiffRow)
+			address := common.HexToAddress("0x0123456789abcdef")
+			mockFetcher = mocks.NewMockStorageFetcher()
+			mockQueue = &mocks.MockStorageQueue{}
+			mockTransformer = &mocks.MockStorageTransformer{Address: address}
+			row = utils.StorageDiffRow{
+				Id:           1337,
+				Contract:     address,
+				BlockHash:    common.HexToHash("0xfedcba9876543210"),
+				BlockHeight:  0,
+				StorageKey:   common.HexToHash("0xabcdef1234567890"),
+				StorageValue: common.HexToHash("0x9876543210abcdef"),
+			}
 		})

-		It("logs error if queuing row fails", func() {
-			address := []byte{1, 2, 3}
-			line := getFakeLine(address)
-			mockTailer := fakes.NewMockTailer()
-			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-			mockQueue := &mocks.MockStorageQueue{}
-			mockQueue.AddError = fakes.FakeError
-			w.Queue = mockQueue
-			keyNotFoundError := utils.ErrStorageKeyNotFound{Key: "unknown_storage_key"}
-			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: keyNotFoundError}
-			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-			tempFile, err := ioutil.TempFile("", "log")
+		It("logs error if fetching storage diffs fails", func(done Done) {
+			mockFetcher.ErrsToReturn = []error{fakes.FakeError}
+			storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
+			storageWatcher.Queue = mockQueue
+			storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
+			tempFile, fileErr := ioutil.TempFile("", "log")
+			Expect(fileErr).NotTo(HaveOccurred())
 			defer os.Remove(tempFile.Name())
-			Expect(err).NotTo(HaveOccurred())
 			logrus.SetOutput(tempFile)

-			assert(func(err error) {
-				Expect(err).NotTo(HaveOccurred())
-				Expect(mockQueue.AddCalled).To(BeTrue())
-				logContent, readErr := ioutil.ReadFile(tempFile.Name())
-				Expect(readErr).NotTo(HaveOccurred())
-				Expect(string(logContent)).To(ContainSubstring(fakes.FakeError.Error()))
-			}, w, mockTailer, []*tail.Line{line})
+			go storageWatcher.Execute(rows, errs, time.Hour)
+
+			Eventually(func() (string, error) {
+				logContent, err := ioutil.ReadFile(tempFile.Name())
+				return string(logContent), err
+			}).Should(ContainSubstring(fakes.FakeError.Error()))
+			close(done)
 		})

-		It("logs any other error", func() {
-			address := []byte{1, 2, 3}
-			line := getFakeLine(address)
-			mockTailer := fakes.NewMockTailer()
-			w := watcher.NewStorageWatcher(mockTailer, test_config.NewTestDB(core.Node{}))
-			executionError := errors.New("storage watcher failed attempting to execute transformer")
-			fakeTransformer := &mocks.MockStorageTransformer{Address: common.BytesToAddress(address), ExecuteErr: executionError}
-			w.AddTransformers([]transformer.StorageTransformerInitializer{fakeTransformer.FakeTransformerInitializer})
-			tempFile, err := ioutil.TempFile("", "log")
-			defer os.Remove(tempFile.Name())
-			Expect(err).NotTo(HaveOccurred())
-			logrus.SetOutput(tempFile)
+		Describe("transforming new storage diffs", func() {
+			BeforeEach(func() {
+				mockFetcher.RowsToReturn = []utils.StorageDiffRow{row}
+				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
+				storageWatcher.Queue = mockQueue
+				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
+			})

-			assert(func(err error) {
-				Expect(err).NotTo(HaveOccurred())
-				logContent, readErr := ioutil.ReadFile(tempFile.Name())
-				Expect(readErr).NotTo(HaveOccurred())
-				Expect(string(logContent)).To(ContainSubstring(executionError.Error()))
-			}, w, mockTailer, []*tail.Line{line})
+			It("executes transformer for recognized storage row", func(done Done) {
+				go storageWatcher.Execute(rows, errs, time.Hour)
+
+				Eventually(func() utils.StorageDiffRow {
+					return mockTransformer.PassedRow
+				}).Should(Equal(row))
+				close(done)
+			})
+
+			It("queues row for later processing if transformer execution fails", func(done Done) {
+				mockTransformer.ExecuteErr = fakes.FakeError
+
+				go storageWatcher.Execute(rows, errs, time.Hour)
+
+				Expect(<-errs).To(BeNil())
+				Eventually(func() bool {
+					return mockQueue.AddCalled
+				}).Should(BeTrue())
+				Eventually(func() utils.StorageDiffRow {
+					return mockQueue.AddPassedRow
+				}).Should(Equal(row))
+				close(done)
+			})
+
+			It("logs error if queueing row fails", func(done Done) {
+				mockTransformer.ExecuteErr = utils.ErrStorageKeyNotFound{}
+				mockQueue.AddError = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Hour)
+
+				Eventually(func() bool {
+					return mockQueue.AddCalled
+				}).Should(BeTrue())
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
 		})
+
+		Describe("transforming queued storage diffs", func() {
+			BeforeEach(func() {
+				mockQueue.RowsToReturn = []utils.StorageDiffRow{row}
+				storageWatcher = watcher.NewStorageWatcher(mockFetcher, test_config.NewTestDB(test_config.NewTestNode()))
+				storageWatcher.Queue = mockQueue
+				storageWatcher.AddTransformers([]transformer.StorageTransformerInitializer{mockTransformer.FakeTransformerInitializer})
+			})
+
+			It("logs error if getting queued storage fails", func(done Done) {
+				mockQueue.GetAllErr = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
+
+			It("executes transformer for storage row", func(done Done) {
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() utils.StorageDiffRow {
+					return mockTransformer.PassedRow
+				}).Should(Equal(row))
+				close(done)
+			})
+
+			It("deletes row from queue if transformer execution successful", func(done Done) {
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(row.Id))
+				close(done)
+			})
+
+			It("logs error if deleting persisted row fails", func(done Done) {
+				mockQueue.DeleteErr = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
+
+			It("deletes obsolete row from queue if contract not recognized", func(done Done) {
+				obsoleteRow := utils.StorageDiffRow{
+					Id:       row.Id + 1,
+					Contract: common.HexToAddress("0xfedcba9876543210"),
+				}
+				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() int {
+					return mockQueue.DeletePassedId
+				}).Should(Equal(obsoleteRow.Id))
+				close(done)
+			})
+
+			It("logs error if deleting obsolete row fails", func(done Done) {
+				obsoleteRow := utils.StorageDiffRow{
+					Id:       row.Id + 1,
+					Contract: common.HexToAddress("0xfedcba9876543210"),
+				}
+				mockQueue.RowsToReturn = []utils.StorageDiffRow{obsoleteRow}
+				mockQueue.DeleteErr = fakes.FakeError
+				tempFile, fileErr := ioutil.TempFile("", "log")
+				Expect(fileErr).NotTo(HaveOccurred())
+				defer os.Remove(tempFile.Name())
+				logrus.SetOutput(tempFile)
+
+				go storageWatcher.Execute(rows, errs, time.Nanosecond)
+
+				Eventually(func() (string, error) {
+					logContent, err := ioutil.ReadFile(tempFile.Name())
+					return string(logContent), err
+				}).Should(ContainSubstring(fakes.FakeError.Error()))
+				close(done)
+			})
+		})
+
 	})
 })
-
-func assert(assertion func(err error), watcher watcher.StorageWatcher, mockTailer *fakes.MockTailer, lines []*tail.Line) {
-	errs := make(chan error, 1)
-	done := make(chan bool, 1)
-	go execute(watcher, errs, done)
-	for _, line := range lines {
-		mockTailer.Lines <- line
-	}
-	close(mockTailer.Lines)
-
-	select {
-	case err := <-errs:
-		assertion(err)
-		break
-	case <-done:
-		assertion(nil)
-		break
-	}
-}
-
-func execute(w watcher.StorageWatcher, errs chan error, done chan bool) {
-	err := w.Execute()
-	if err != nil {
-		errs <- err
-	} else {
-		done <- true
-	}
-}
-
-func getFakeLine(address []byte) *tail.Line {
-	blockHash := []byte{4, 5, 6}
-	blockHeight := int64(789)
-	storageKey := []byte{9, 8, 7}
-	storageValue := []byte{6, 5, 4}
-	return &tail.Line{
-		Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address), common.Bytes2Hex(blockHash), blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)),
-		Time: time.Time{},
-		Err:  nil,
-	}
-}
@@ -3,12 +3,15 @@ package main
 import (
 	"github.com/vulcanize/vulcanizedb/cmd"

-	log "github.com/sirupsen/logrus"
 	"os"
+
+	log "github.com/sirupsen/logrus"
 )

 func main() {
-	log.SetFormatter(&log.JSONFormatter{})
+	log.SetFormatter(&log.JSONFormatter{
+		PrettyPrint: true,
+	})
 	file, err := os.OpenFile("vulcanizedb.log",
 		os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 	if err == nil {
@@ -164,7 +164,7 @@ func (tr *Transformer) Init() error {
 // Iterates through stored, initialized contract objects
 // Iterates through contract's event filters, grabbing watched event logs
 // Uses converter to convert logs into custom log type
-// Persists converted logs into custuom postgres tables
+// Persists converted logs into custom postgres tables
 // Calls selected methods, using token holder address generated during event log conversion
 func (tr *Transformer) Execute() error {
 	if len(tr.Contracts) == 0 {
@@ -39,7 +39,7 @@ var _ = Describe("Transformer", func() {

 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			blockRetriever := &fakes.MockFullBlockRetriever{}
+			blockRetriever := &fakes.MockFullSyncBlockRetriever{}
 			firstBlock := int64(1)
 			mostRecentBlock := int64(2)
 			blockRetriever.FirstBlock = firstBlock
@@ -74,7 +74,7 @@ var _ = Describe("Transformer", func() {
 		})

 		It("Fails to initialize if first and most recent blocks cannot be fetched from vDB", func() {
-			blockRetriever := &fakes.MockFullBlockRetriever{}
+			blockRetriever := &fakes.MockFullSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})

@@ -27,7 +27,7 @@ import (

 func TestConverter(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Converter Suite Test")
+	RunSpecs(t, "Header Sync Converter Suite Test")
 }

 var _ = BeforeSuite(func() {
@@ -22,7 +22,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
@@ -22,7 +22,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/fetcher"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 )
@@ -23,7 +23,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -32,7 +32,7 @@ import (

 var _ = Describe("Repository", func() {
 	var db *postgres.DB
-	var contractHeaderRepo repository.HeaderRepository // contract_watcher light header repository
+	var contractHeaderRepo repository.HeaderRepository // contract_watcher headerSync header repository
 	var coreHeaderRepo repositories.HeaderRepository   // pkg/datastore header repository
 	var eventIDs = []string{
 		"eventName_contractAddr",
@@ -27,7 +27,7 @@ import (

 func TestRepository(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Repository Suite Test")
+	RunSpecs(t, "Header Sync Repository Suite Test")
 }

 var _ = BeforeSuite(func() {
@@ -20,7 +20,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -27,7 +27,7 @@ import (

 func TestRetriever(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Block Number Retriever Suite Test")
+	RunSpecs(t, "Header Sync Block Number Retriever Suite Test")
 }

 var _ = BeforeSuite(func() {
@@ -24,10 +24,10 @@ import (
 	gethTypes "github.com/ethereum/go-ethereum/core/types"

 	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/fetcher"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/poller"
@@ -37,7 +37,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )

-// Requires a light synced vDB (headers) and a running eth node (or infura)
+// Requires a header synced vDB (headers) and a running eth node (or infura)
 type Transformer struct {
 	// Database interfaces
 	EventRepository  srep.EventRepository        // Holds transformed watched event log data
@@ -77,14 +77,14 @@ type Transformer struct {
 func NewTransformer(con config.ContractConfig, bc core.BlockChain, db *postgres.DB) *Transformer {

 	return &Transformer{
-		Poller:           poller.NewPoller(bc, db, types.LightSync),
+		Poller:           poller.NewPoller(bc, db, types.HeaderSync),
 		Fetcher:          fetcher.NewFetcher(bc),
 		Parser:           parser.NewParser(con.Network),
 		HeaderRepository: repository.NewHeaderRepository(db),
 		Retriever:        retriever.NewBlockRetriever(db),
 		Converter:        &converter.Converter{},
 		Contracts:        map[string]*contract.Contract{},
-		EventRepository:  srep.NewEventRepository(db, types.LightSync),
+		EventRepository:  srep.NewEventRepository(db, types.HeaderSync),
 		Config:           con,
 	}
 }
@@ -27,7 +27,7 @@ import (

 func TestTransformer(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Light Transformer Suite Test")
+	RunSpecs(t, "Header Sync Transformer Suite Test")
 }

 var _ = BeforeSuite(func() {
@@ -20,8 +20,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"

-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/retriever"
-	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/transformer"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/retriever"
+	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/parser"
@@ -33,7 +33,7 @@ var _ = Describe("Transformer", func() {
 	var fakeAddress = "0x1234567890abcdef"
 	Describe("Init", func() {
 		It("Initializes transformer's contract objects", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			firstBlock := int64(1)
 			blockRetriever.FirstBlock = firstBlock

@@ -61,7 +61,7 @@ var _ = Describe("Transformer", func() {
 		})

 		It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})

@@ -74,7 +74,7 @@ var _ = Describe("Transformer", func() {

 	Describe("Execute", func() {
 		It("Executes contract transformations", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			firstBlock := int64(1)
 			blockRetriever.FirstBlock = firstBlock

@@ -102,7 +102,7 @@ var _ = Describe("Transformer", func() {
 		})

 		It("Fails to initialize if first block cannot be fetched from vDB headers table", func() {
-			blockRetriever := &fakes.MockLightBlockRetriever{}
+			blockRetriever := &fakes.MockHeaderSyncBlockRetriever{}
 			blockRetriever.FirstBlockErr = fakes.FakeError
 			t := getFakeTransformer(blockRetriever, &fakes.MockParser{}, &fakes.MockPoller{})

@@ -119,7 +119,7 @@ func getFakeTransformer(blockRetriever retriever.BlockRetriever, parsr parser.Pa
 		Parser:           parsr,
 		Retriever:        blockRetriever,
 		Poller:           pollr,
-		HeaderRepository: &fakes.MockLightHeaderRepository{},
+		HeaderRepository: &fakes.MockHeaderSyncHeaderRepository{},
 		Contracts:        map[string]*contract.Contract{},
 		Config:           mocks.MockConfig,
 	}
@@ -59,7 +59,7 @@ type NewOwnerLog struct {
 	Owner          string `db:"owner_"`
 }

-type LightTransferLog struct {
+type HeaderSyncTransferLog struct {
 	Id        int64  `db:"id"`
 	HeaderID  int64  `db:"header_id"`
 	TokenName string `db:"token_name"`
@@ -71,7 +71,7 @@ type LightTransferLog struct {
 	RawLog    []byte `db:"raw_log"`
 }

-type LightNewOwnerLog struct {
+type HeaderSyncNewOwnerLog struct {
 	Id        int64  `db:"id"`
 	HeaderID  int64  `db:"header_id"`
 	TokenName string `db:"token_name"`
@@ -240,13 +240,13 @@ func TearDown(db *postgres.DB) {
 	_, err = tx.Exec(`DELETE FROM full_sync_transactions`)
 	Expect(err).NotTo(HaveOccurred())

-	_, err = tx.Exec("DELETE FROM light_sync_transactions")
+	_, err = tx.Exec("DELETE FROM header_sync_transactions")
 	Expect(err).NotTo(HaveOccurred())

 	_, err = tx.Exec(`DELETE FROM full_sync_receipts`)
 	Expect(err).NotTo(HaveOccurred())

-	_, err = tx.Exec(`DELETE FROM light_sync_receipts`)
+	_, err = tx.Exec(`DELETE FROM header_sync_receipts`)
 	Expect(err).NotTo(HaveOccurred())

 	_, err = tx.Exec(`DROP TABLE checked_headers`)
@@ -258,13 +258,13 @@ func TearDown(db *postgres.DB) {
 	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
 	Expect(err).NotTo(HaveOccurred())

-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
+	_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e CASCADE`)
 	Expect(err).NotTo(HaveOccurred())

 	_, err = tx.Exec(`DROP SCHEMA IF EXISTS full_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
 	Expect(err).NotTo(HaveOccurred())

-	_, err = tx.Exec(`DROP SCHEMA IF EXISTS light_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
+	_, err = tx.Exec(`DROP SCHEMA IF EXISTS header_0x314159265dd8dbb310642f98f50c066173c1259b CASCADE`)
 	Expect(err).NotTo(HaveOccurred())

 	err = tx.Commit()
@@ -220,7 +220,7 @@ var _ = Describe("Parser", func() {
 			Expect(err).ToNot(HaveOccurred())

 			selectMethods := p.GetMethods([]string{})
-			Expect(len(selectMethods)).To(Equal(22))
+			Expect(len(selectMethods)).To(Equal(25))
 		})
 	})
 })
@@ -84,8 +84,8 @@ func (r *eventRepository) PersistLogs(logs []types.Log, eventInfo types.Event, c
 func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
 	var err error
 	switch r.mode {
-	case types.LightSync:
-		err = r.persistLightSyncLogs(logs, eventInfo, contractAddr, contractName)
+	case types.HeaderSync:
+		err = r.persistHeaderSyncLogs(logs, eventInfo, contractAddr, contractName)
 	case types.FullSync:
 		err = r.persistFullSyncLogs(logs, eventInfo, contractAddr, contractName)
 	default:
@@ -95,8 +95,8 @@ func (r *eventRepository) persistLogs(logs []types.Log, eventInfo types.Event, c
 	return err
 }

-// Creates a custom postgres command to persist logs for the given event (compatible with light synced vDB)
-func (r *eventRepository) persistLightSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
+// Creates a custom postgres command to persist logs for the given event (compatible with header synced vDB)
+func (r *eventRepository) persistHeaderSyncLogs(logs []types.Log, eventInfo types.Event, contractAddr, contractName string) error {
 	tx, err := r.db.Beginx()
 	if err != nil {
 		return err
@@ -232,7 +232,7 @@ func (r *eventRepository) newEventTable(tableID string, event types.Event) error
 			pgStr = pgStr + fmt.Sprintf(" %s_ %s NOT NULL,", strings.ToLower(field.Name), field.PgType)
 		}
 		pgStr = pgStr + " CONSTRAINT log_index_fk FOREIGN KEY (vulcanize_log_id) REFERENCES logs (id) ON DELETE CASCADE)"
-	case types.LightSync:
+	case types.HeaderSync:
 		pgStr = pgStr + "(id SERIAL, header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE, token_name CHARACTER VARYING(66) NOT NULL, raw_log JSONB, log_idx INTEGER NOT NULL, tx_idx INTEGER NOT NULL,"

 		for _, field := range event.Fields {
@@ -27,8 +27,8 @@ import (
 	. "github.com/onsi/gomega"

 	fc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/converter"
-	lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/converter"
-	lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/light/repository"
+	lc "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/converter"
+	lr "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"
@@ -208,9 +208,9 @@ var _ = Describe("Repository", func() {
 		})
 	})

-	Describe("Light sync mode", func() {
+	Describe("Header sync mode", func() {
 		BeforeEach(func() {
-			dataStore = sr.NewEventRepository(db, types.LightSync)
+			dataStore = sr.NewEventRepository(db, types.HeaderSync)
 		})

 		Describe("CreateContractSchema", func() {
@@ -242,7 +242,7 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(created).To(Equal(true))

-				tableID := fmt.Sprintf("%s_%s.%s_event", types.LightSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
+				tableID := fmt.Sprintf("%s_%s.%s_event", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(event.Name))
 				_, ok := dataStore.CheckTableCache(tableID)
 				Expect(ok).To(Equal(false))

@@ -292,12 +292,12 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())

 				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM light_%s.transfer_event", constants.TusdContractAddress))
+				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
 				Expect(err).ToNot(HaveOccurred())
 				Expect(count).To(Equal(2))

-				scanLog := test_helpers.LightTransferLog{}
-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
+				scanLog := test_helpers.HeaderSyncTransferLog{}
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event LIMIT 1", constants.TusdContractAddress)).StructScan(&scanLog)
 				Expect(err).ToNot(HaveOccurred())
 				Expect(scanLog.HeaderID).To(Equal(headerID))
 				Expect(scanLog.TokenName).To(Equal("TrueUSD"))
@@ -334,7 +334,7 @@ var _ = Describe("Repository", func() {

 				// Show that no new logs were entered
 				var count int
-				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM light_%s.transfer_event", constants.TusdContractAddress))
+				err = db.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM header_%s.transfer_event", constants.TusdContractAddress))
 				Expect(err).ToNot(HaveOccurred())
 				Expect(count).To(Equal(2))
 			})
@@ -149,9 +149,9 @@ var _ = Describe("Repository", func() {
 		})
 	})

-	Describe("Light Sync Mode", func() {
+	Describe("Header Sync Mode", func() {
 		BeforeEach(func() {
-			dataStore = repository.NewMethodRepository(db, types.LightSync)
+			dataStore = repository.NewMethodRepository(db, types.HeaderSync)
 		})

 		Describe("CreateContractSchema", func() {
@@ -199,7 +199,7 @@ var _ = Describe("Repository", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(created).To(Equal(true))

-				tableID := fmt.Sprintf("%s_%s.%s_method", types.LightSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
+				tableID := fmt.Sprintf("%s_%s.%s_method", types.HeaderSync, strings.ToLower(con.Address), strings.ToLower(method.Name))
 				_, ok := dataStore.CheckTableCache(tableID)
 				Expect(ok).To(Equal(false))

@@ -214,13 +214,13 @@ var _ = Describe("Repository", func() {
 		})

 		Describe("PersistResult", func() {
-			It("Persists result from method polling in custom pg table for light sync mode vDB", func() {
+			It("Persists result from method polling in custom pg table for header sync mode vDB", func() {
 				err = dataStore.PersistResults([]types.Result{mockResult}, method, con.Address, con.Name)
 				Expect(err).ToNot(HaveOccurred())

 				scanStruct := test_helpers.BalanceOf{}

-				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM light_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
+				err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.balanceof_method", constants.TusdContractAddress)).StructScan(&scanStruct)
 				expectedLog := test_helpers.BalanceOf{
 					Id:        1,
 					TokenName: "TrueUSD",
@@ -38,14 +38,14 @@ type Field struct {

 // Struct to hold instance of an event log data
 type Log struct {
-	Id     int64             // VulcanizeIdLog for full sync and header ID for light sync contract watcher
+	Id     int64             // VulcanizeIdLog for full sync and header ID for header sync contract watcher
 	Values map[string]string // Map of event input names to their values

 	// Used for full sync only
 	Block int64
 	Tx    string

-	// Used for lightSync only
+	// Used for headerSync only
 	LogIndex         uint
 	TransactionIndex uint
 	Raw              []byte // json.Unmarshalled byte array of geth/core/types.Log{}
@@ -21,18 +21,18 @@ import "fmt"
 type Mode int

 const (
-	LightSync Mode = iota
+	HeaderSync Mode = iota
 	FullSync
 )

 func (mode Mode) IsValid() bool {
-	return mode >= LightSync && mode <= FullSync
+	return mode >= HeaderSync && mode <= FullSync
 }

 func (mode Mode) String() string {
 	switch mode {
-	case LightSync:
-		return "light"
+	case HeaderSync:
+		return "header"
 	case FullSync:
 		return "full"
 	default:
@@ -42,23 +42,23 @@ func (mode Mode) String() string {

 func (mode Mode) MarshalText() ([]byte, error) {
 	switch mode {
-	case LightSync:
-		return []byte("light"), nil
+	case HeaderSync:
+		return []byte("header"), nil
 	case FullSync:
 		return []byte("full"), nil
 	default:
-		return nil, fmt.Errorf("contract watcher: unknown mode %d, want LightSync or FullSync", mode)
+		return nil, fmt.Errorf("contract watcher: unknown mode %d, want HeaderSync or FullSync", mode)
 	}
 }

 func (mode *Mode) UnmarshalText(text []byte) error {
 	switch string(text) {
-	case "light":
-		*mode = LightSync
+	case "header":
+		*mode = HeaderSync
 	case "full":
 		*mode = FullSync
 	default:
-		return fmt.Errorf(`contract watcher: unknown mode %q, want "light" or "full"`, text)
+		return fmt.Errorf(`contract watcher: unknown mode %q, want "header" or "full"`, text)
 	}
 	return nil
 }
@@ -29,7 +29,7 @@ type Block struct {
 	Number       int64  `db:"number"`
 	ParentHash   string `db:"parent_hash"`
 	Size         string `db:"size"`
-	Time         int64  `db:"time"`
+	Time         uint64 `db:"time"`
 	Transactions []TransactionModel
 	UncleHash    string `db:"uncle_hash"`
 	UnclesReward string `db:"uncles_reward"`
@@ -30,7 +30,7 @@ type BlockChain interface {
 	GetBlockByNumber(blockNumber int64) (Block, error)
 	GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error)
 	GetHeaderByNumber(blockNumber int64) (Header, error)
-	GetHeaderByNumbers(blockNumbers []int64) ([]Header, error)
+	GetHeadersByNumbers(blockNumbers []int64) ([]Header, error)
 	GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)
 	GetTransactions(transactionHashes []common.Hash) ([]TransactionModel, error)
 	LastBlock() (*big.Int, error)
@@ -42,7 +42,7 @@ type POAHeader struct {
 	Number      *hexutil.Big   `json:"number"           gencodec:"required"`
 	GasLimit    hexutil.Uint64 `json:"gasLimit"         gencodec:"required"`
 	GasUsed     hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
-	Time        *hexutil.Big   `json:"timestamp"        gencodec:"required"`
+	Time        hexutil.Uint64 `json:"timestamp"        gencodec:"required"`
 	Extra       hexutil.Bytes  `json:"extraData"        gencodec:"required"`
 	Hash        common.Hash    `json:"hash"`
 }
@@ -21,6 +21,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
 )

 type Reader interface {
@@ -48,7 +49,7 @@ func (ldbr *LevelDatabaseReader) GetBlockNumber(hash common.Hash) *uint64 {
 }

 func (ldbr *LevelDatabaseReader) GetBlockReceipts(hash common.Hash, number uint64) types.Receipts {
-	return rawdb.ReadReceipts(ldbr.reader, hash, number)
+	return rawdb.ReadReceipts(ldbr.reader, hash, number, &params.ChainConfig{})
 }

 func (ldbr *LevelDatabaseReader) GetCanonicalHash(number uint64) common.Hash {
@@ -80,7 +80,7 @@ var _ = Describe("Saving blocks", func() {
 		blockNonce := "0x881db2ca900682e9a9"
 		miner := "x123"
 		extraData := "xextraData"
-		blockTime := int64(1508981640)
+		blockTime := uint64(1508981640)
 		uncleHash := "x789"
 		blockSize := string("1000")
 		difficulty := int64(10)
@@ -98,7 +98,7 @@ var _ = Describe("Saving blocks", func() {
 			Number:       blockNumber,
 			ParentHash:   blockParentHash,
 			Size:         blockSize,
-			Time:         blockTime,
+			Time:         uint64(blockTime),
 			UncleHash:    uncleHash,
 			UnclesReward: unclesReward,
 		}
@@ -54,7 +54,7 @@ func (repository HeaderRepository) CreateOrUpdateHeader(header core.Header) (int

 func (repository HeaderRepository) CreateTransactions(headerID int64, transactions []core.TransactionModel) error {
 	for _, transaction := range transactions {
-		_, err := repository.database.Exec(`INSERT INTO public.light_sync_transactions
+		_, err := repository.database.Exec(`INSERT INTO public.header_sync_transactions
 		(header_id, hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value") 
 		VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC)
 		ON CONFLICT DO NOTHING`, headerID, transaction.Hash, transaction.GasLimit, transaction.GasPrice,
@@ -69,7 +69,7 @@ func (repository HeaderRepository) CreateTransactions(headerID int64, transactio

 func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID int64, transaction core.TransactionModel) (int64, error) {
 	var txId int64
-	err := tx.QueryRowx(`INSERT INTO public.light_sync_transactions
+	err := tx.QueryRowx(`INSERT INTO public.header_sync_transactions
 		(header_id, hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value")
 		VALUES ($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6::NUMERIC, $7, $8, $9::NUMERIC, $10, $11::NUMERIC)
 		ON CONFLICT (header_id, hash) DO UPDATE 
@@ -87,7 +87,7 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i

 func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) {
 	var receiptId int64
-	err := tx.QueryRowx(`INSERT INTO public.light_sync_receipts
+	err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
               (header_id, transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
               VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
 			   ON CONFLICT (header_id, transaction_id) DO UPDATE
@@ -228,7 +228,7 @@ var _ = Describe("Block header repository", func() {
 			var dbReceipt idModel
 			err = db.Get(&dbReceipt,
 				`SELECT transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
-				FROM public.light_sync_receipts WHERE header_id = $1`, headerID)
+				FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbReceipt.TransactionId).To(Equal(txId))
 			Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
@@ -288,7 +288,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransactions []core.TransactionModel
 			err = db.Select(&dbTransactions,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbTransactions).To(ConsistOf(transactions))
 		})
@@ -300,7 +300,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransactions []core.TransactionModel
 			err = db.Select(&dbTransactions,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(dbTransactions)).To(Equal(2))
 		})
@@ -337,7 +337,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransaction core.TransactionModel
 			err = db.Get(&dbTransaction,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(dbTransaction).To(Equal(transaction))
 		})
@@ -381,7 +381,7 @@ var _ = Describe("Block header repository", func() {
 			var dbTransactions []core.TransactionModel
 			err = db.Select(&dbTransactions,
 				`SELECT hash, gas_limit, gas_price, input_data, nonce, raw, tx_from, tx_index, tx_to, "value"
-				FROM public.light_sync_transactions WHERE header_id = $1`, headerID)
+				FROM public.header_sync_transactions WHERE header_id = $1`, headerID)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(dbTransactions)).To(Equal(1))
 		})
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Gabe Laughlin	baea9740c1	Merge pull request #125 from vulcanize/decode-packed-addresses (VDB-768) Allow addresses in packed slots to be decoded	2019-08-05 21:40:15 -05:00
Gabe Laughlin	4f8b057726	(VDB-768) Allow addresses in packed slots to be decoded	2019-08-01 22:40:50 -05:00
Andrew J Yao	65e762f80b	Merge pull request #124 from vulcanize/refactor-mcd-configs Removes _checked from column name query	2019-07-30 18:18:42 -07:00
Andrew J Yao	dc8d52a58d	Removes _checked from column name query	2019-07-30 18:07:25 -07:00
Rob Mulholand	f2547b5a42	Merge pull request #117 from vulcanize/geth-1.9.0 Update Geth to 1.9.0	2019-07-29 15:04:37 -05:00
Rob Mulholand	62e454b7fe	Update docs to reflect using go mod	2019-07-23 15:28:08 -05:00
Rob Mulholand	ca4451d914	Update Travis to enable go mod - set GO111MODULE=on - use go 1.12	2019-07-23 15:28:08 -05:00
Rob Mulholand	f51f5c64a0	Replace dep with go mod	2019-07-23 15:28:08 -05:00
Rob Mulholand	987abd4b2e	Update Geth to 1.9.0	2019-07-23 15:26:18 -05:00
Andrew J Yao	a3b55bfd56	Merge pull request #120 from vulcanize/logging-enhancement Logging enhancement	2019-07-23 13:22:58 -07:00
Andrew J Yao	ee77fc6521	Remove injection of SubCommand for logs	2019-07-23 13:11:20 -07:00
Elizabeth	85aba29cf2	Merge pull request #121 from vulcanize/allow-for-packed-storage-slots VDB-318 Allow for packed storage slots	2019-07-23 13:48:36 -05:00
Elizabeth Engelman	2c66afbcd7	Address PR feedback - factor out bitsPerByte constant - panic if the necessary args aren't passed to GetStorageValueMetadataForPackedSlot - only have one decode integer method in the deocoder file	2019-07-23 13:36:58 -05:00
Andrew J Yao	a188e1dd79	Incorporate changes from Edvard PR	2019-07-23 08:10:58 -07:00
Andrew J Yao	92d153b010	Add subCommand name to log fields	2019-07-23 08:09:57 -07:00
Andrew J Yao	d7a82e353b	Add calling method to debug level log messages	2019-07-23 08:08:15 -07:00
Elizabeth Engelman	69ad521482	Apply go fmt changes and cleanup	2019-07-19 10:44:36 -05:00
Elizabeth Engelman	d30b4ea4f2	Update GetStorageValueMetadata	2019-07-19 10:13:30 -05:00
Elizabeth Engelman	51915f7506	Pass storage values to repo.create as a map	2019-07-18 18:40:38 -05:00
Elizabeth Engelman	ebfb4965c7	Add tests for decoding packed storage	2019-07-18 16:21:57 -05:00
Elizabeth Engelman	b0fff9a1dd	First pass at decoded packed storage slots	2019-07-18 13:30:04 -05:00
Edvard Hübinette	2c092e8d04	[VDB-751 VDB-754] Bugfix null pointer panic and improve logging (#119 ) * VDB-751 VDB-754 Bugfix null pointer panic and improve logging * Fix typo	2019-07-18 09:21:40 +02:00
Rob Mulholand	af6190ea09	Merge pull request #113 from vulcanize/plugin-updates Plugin updates	2019-06-27 14:11:12 -05:00
Rob Mulholand	0f95267e84	Use goose library for running plugin migrations - Removes dependency on goose binary existing in plugin execution environment	2019-06-27 13:58:58 -05:00
Rob Mulholand	f0bcf930f4	Separate tmp migrations dir from home in join - enable separator between home and migrations dir	2019-06-27 13:58:58 -05:00
Rob Mulholand	0558792282	Log error on failure to setup plugin migrations - include specific error	2019-06-27 13:58:58 -05:00
Rob Mulholand	62cca62432	Log error on failure to read config file - include actual error to highlight the specific issue	2019-06-27 13:58:58 -05:00
Andrew J Yao	7092d33338	Merge pull request #112 from vulcanize/vdb-605-add-debug-logging Adds ability to set log level from toml or CLI args	2019-06-27 11:56:47 -07:00
Andrew J Yao	11a65c525c	Adds ability to set log level from toml or CLI args	2019-06-27 11:14:11 -07:00
Elizabeth	3d57f6030d	Merge pull request #106 from vulcanize/update-docs Remove repository-maintenance.md as this info now lives in contributi…	2019-06-26 13:45:54 -05:00
Elizabeth Engelman	b124bea974	Remove repository-maintenance.md as this info now lives in contributing.md	2019-06-18 14:42:20 -05:00
Rob Mulholand	f0a7a7d999	Merge pull request #103 from vulcanize/remove-queued-storage-duplicates Don't duplicate queued storage diffs	2019-06-17 15:55:35 -05:00
Rob Mulholand	7d6629e5b4	Merge pull request #102 from vulcanize/fix-mappings-util Fix mappings util	2019-06-17 15:45:55 -05:00
Elizabeth	d761f88b30	Merge pull request #101 from vulcanize/update-transformer-execute VDB-696 Update transformer execute	2019-06-17 15:19:51 -05:00
Elizabeth Engelman	948246cf83	Address PR comments	2019-06-17 14:49:25 -05:00
Rob Mulholand	e11f2c8c59	Don't duplicate queued storage diffs - currently, if we don't recognize the same diff several times (e.g. if you restart the storage diff watcher pointed at the same file), we'll add the same row to the queue on each run. - these changes assure we only queue an unrecognized diff once.	2019-06-14 11:26:15 -05:00
Rob Mulholand	95fa6280c5	Fix mappings util - Remove hex prefix so that the function can accept keys with or without a hex prefix and return the same result	2019-06-13 18:33:22 -05:00
Elizabeth Engelman	fd126c7bea	Remove unused RecheckHeaders method and update Repo tests	2019-06-13 15:19:23 -05:00
Elizabeth Engelman	f292888195	Refactor checked column specific SQL	2019-06-13 12:36:29 -05:00
Elizabeth Engelman	adf91df23a	Remove unused RecheckHeaders method from the events Repository interface	2019-06-13 10:33:53 -05:00
Elizabeth Engelman	ede6ad1c8f	Remove unused MissingHeaders method from the events Repository interface	2019-06-13 10:33:48 -05:00
Elizabeth Engelman	ee13d5c1df	Remove unused recheckHeaders bool argument from Transformer.Execute	2019-06-13 10:33:16 -05:00
Elizabeth	eb2b3a0263	Merge pull request #98 from vulcanize/additional-doc-updates Move changelog and diagram information to contributing.md	2019-06-03 14:24:43 -05:00
Elizabeth Engelman	5088574437	Address PR comment	2019-06-03 13:36:56 -05:00
Elizabeth Engelman	d61f4f3cb4	Move changelog and diagram information to contributing.md	2019-06-03 13:31:50 -05:00
Elizabeth	1a455741ce	Merge pull request #97 from vulcanize/quick-link-fix Fixes broken link in docs	2019-06-03 13:30:51 -05:00
yaoandrew	180c2d963e	Fixes broken link in docs	2019-05-28 15:24:04 -07:00
Rob Mulholand	2598704690	Merge pull request #96 from vulcanize/remove-storage-diff-duplicate-handling Remove handling of duplicate storage diffs in watcher	2019-05-20 14:04:50 -05:00
Rob Mulholand	e2909797fc	Remove handling of duplicate storage diffs in watcher - Can push this responsibility down to the transformers - Update docs to reflect that transformers should handle duplicates	2019-05-20 13:29:09 -05:00
Elizabeth	79765c7998	Merge pull request #94 from vulcanize/documentation-updates Documentation updates	2019-05-15 13:22:15 -05:00
Gabe Laughlin	9c6549258e	Merge pull request #95 from vulcanize/vdb-479-upgrade-postgres (VDB-479) Upgrade Posgres to 11	2019-05-15 12:29:06 -05:00
Gabe Laughlin	b4cdf76905	(VDB-479) Upgrade Posgres to 11	2019-05-14 16:49:05 -05:00
Elizabeth Engelman	622ea44a6f	Mention reorgs in data-sync documentation	2019-05-13 16:34:00 -05:00
Elizabeth Engelman	d947c8f30a	Rename sync command to fullSync	2019-05-13 16:14:56 -05:00
Elizabeth Engelman	481988cb08	Update date-syncing doc	2019-05-13 16:14:56 -05:00
Elizabeth Engelman	4580f0fc2e	Updates to custom-transformers doc	2019-05-13 16:14:56 -05:00
Elizabeth Engelman	eba52442ec	Restructure README to conform with standard readme spec	2019-05-13 16:14:56 -05:00
Elizabeth Engelman	7ddf728db2	Descript different custom sync transformer types	2019-05-13 13:40:46 -05:00
Elizabeth Engelman	829a581727	Rename sync documentation file	2019-05-13 13:31:23 -05:00
Elizabeth Engelman	fa03716cb2	Address small PR comments	2019-05-10 11:54:15 -05:00
Elizabeth Engelman	bd3e841e21	Update links in README to be relative to current branch	2019-05-10 11:01:17 -05:00
Elizabeth Engelman	8dc9ddf65e	Add Code of Conduct	2019-05-10 10:07:25 -05:00
Elizabeth Engelman	7d1b334bda	Update postgraphile documentation to mention no-ignore-rbac flag	2019-05-10 09:50:28 -05:00
Elizabeth Engelman	ade1429ce3	Update transformer documentation broken into generic and custom	2019-05-08 17:29:24 -05:00
Elizabeth Engelman	5d1ba5903c	Update contributing guidelines	2019-05-08 13:42:43 -05:00
Elizabeth Engelman	a49f5d7582	Add repository maintenance documentation	2019-05-08 13:42:43 -05:00
Elizabeth Engelman	436d9b9cb1	Add VDB overview diagram to README	2019-05-08 13:42:43 -05:00
Elizabeth Engelman	f7d520c933	Update header sync transformer alias in ContractWatcher	2019-05-08 13:42:43 -05:00
Elizabeth Engelman	e1a0d894a2	Update blockchain method GetHeaderByNumbers -> GetHeadersByNumbers	2019-05-08 13:42:43 -05:00
Elizabeth Engelman	ba81766f6c	Small spelling fix	2019-05-08 13:42:43 -05:00
Elizabeth Engelman	b96f6c4d31	Various README updates	2019-05-08 13:42:43 -05:00
Rob Mulholand	c5cfa85f4c	Merge pull request #90 from vulcanize/vdb-563-remove-postgraphile (VDB-563) Remove Postgraphile bespoke code	2019-05-08 12:51:56 -05:00
Rob Mulholand	702e839f5b	Update Travis to reflect removing Postgraphile	2019-05-08 09:11:51 -05:00
Rob Mulholand	ca354de229	(VDB-563) Remove Postgraphile bespoke code - Add documentation for running Postgraphile from their binary	2019-05-08 09:11:51 -05:00
Rob Mulholand	3b531950f2	Merge pull request #93 from vulcanize/vdb-570-storage-duplicates (VDB-570) Handle duplicate storage diffs	2019-05-02 13:40:44 -05:00
Rob Mulholand	36c4da372b	(VDB-570) Handle duplicate storage diffs - If processing a new diff for a row that already exists in the DB, ignore the error without logging or queueing - If processing a queued diff for a row that already exists, remove it from the queue	2019-05-02 11:39:16 -05:00
ana0	202fb6e6a6	Merge pull request #89 from vulcanize/feature/configurableMutations Disable default mutations is now in config file	2019-05-02 12:36:36 -04:00
Elizabeth	77b7ee9e4c	Merge pull request #91 from vulcanize/vdb-558-remove-log-note-transformer Remove LogNote Transformer and Converter	2019-05-02 11:26:10 -05:00
anon	754c8cb18d	Config path name change	2019-05-02 09:55:25 -04:00
Rob Mulholand	782e3fd095	Merge pull request #86 from vulcanize/vdb-371-recheck-queued-storage (VDB-371) recheck queued storage	2019-05-01 12:49:02 -05:00
Rob Mulholand	6716c3b92a	Make queue recheck interval configurable via CLI	2019-05-01 12:32:39 -05:00
Rob Mulholand	b036053937	Queue storage diffs if transformer execution fails - For any error, not just if key isn't recognized - Means we don't lose track of diffs on random ephemeral errors	2019-05-01 12:32:39 -05:00
Rob Mulholand	76ab914bdc	Add license	2019-05-01 12:32:39 -05:00
Rob Mulholand	d77f3fe012	Don't pass empty row to channel on error	2019-05-01 12:30:37 -05:00
Rob Mulholand	6a86de87b4	(VDB-371) Recheck queued storage - Iterate through queued storage at defined interval, popping rows from the queue if successfully persisted	2019-05-01 12:30:37 -05:00
Rob Mulholand	bf4b1687a0	Add get/delete functions to storage queue	2019-05-01 12:30:37 -05:00
Rob Mulholand	2d684c5aec	Extract storage diff fetching behind an interface - Replaces directly reading from a CSV - Simplifies testing - Should hopefully make it easier to plug in other sources for storage diffs (e.g. differently formatted CSVs, JSON RPC, etc)	2019-05-01 12:30:36 -05:00
Gabe Laughlin	c93e6adb27	Merge pull request #92 from vulcanize/vdb-560-rename-lightsync-again (VDB-560) Rename lightSync to headerSync	2019-05-01 12:20:33 -05:00
Gabe Laughlin	a344432156	(VDB-560) Rename lightSync to headerSync	2019-05-01 12:12:55 -05:00
Elizabeth Engelman	eb1a9cb381	Remove LogNote Transformer and Converter	2019-04-30 12:25:51 -05:00
anon	bc3f0d5eaf	Disable default mutations is now in config file	2019-04-29 17:18:34 -04:00
Ian Norden	caba9e91d3	Merge pull request #85 from vulcanize/docs add some info for separate compose and execute commands	2019-04-21 12:46:21 -05:00
Ian Norden	01ffc80ef2	add some info for separate compose and execute commands; note about flaky test	2019-04-19 14:49:09 -05:00
Rob Mulholand	9c06e9387c	Merge pull request #82 from vulcanize/fixup-transactions-syncing Don't lookup transactions if no log events	2019-04-18 10:11:45 -05:00
Rob Mulholand	3fd6269b78	Don't lookup transactions if no log events - Prevents EOF error on transactions lookup	2019-04-18 09:53:18 -05:00
Rob Mulholand	13cd7126cc	Merge pull request #84 from vulcanize/improve-postgraphile-interaction Add support for restricted graphql users	2019-04-18 09:51:33 -05:00
Edvard	7d2542aa19	Fix typo	2019-04-18 15:57:38 +02:00
Edvard	355f245bfe	Config parameter rename	2019-04-18 13:56:28 +02:00
Edvard	ec796d9813	Remove old deps	2019-04-18 12:59:51 +02:00
Edvard	a1adf0e7fc	Add support for restricted graphql user	2019-04-18 11:04:14 +02:00
ana0	dbd2208d31	Merge pull request #79 from vulcanize/postgraphileConfig Postgraphile config	2019-04-16 14:29:19 -04:00
anon	f99e397699	Add schemas to config	2019-04-15 14:17:33 -04:00
ana0	0f425ac76b	Merge pull request #78 from vulcanize/databaseConfig Adds user param to the Makefile and MigrationManager	2019-04-11 16:22:50 -04:00
ana0	964fc8bd22	Merge branch 'staging' into databaseConfig	2019-04-11 14:19:57 -04:00
anon	4e8485810e	Add note to readme	2019-04-11 13:14:04 -04:00
anon	aac2d852ca	Default user is postgres	2019-04-10 18:18:58 -04:00
anon	2a9b7b1414	Don't change default toml	2019-04-10 17:59:23 -04:00
anon	bd12149594	Database config	2019-04-10 17:56:31 -04:00
Gabe Laughlin	8b704e9f1e	Merge pull request #76 from vulcanize/fake-header-customization Add timestamp customization for fake header	2019-04-10 09:09:48 -05:00
Ian Norden	fcaa28d923	readme addition; fix postgraphile lodash and webpack-dev-server alerts	2019-04-09 19:03:08 -05:00
Gabe Laughlin	8903b6fb2d	Add timestamp customization for fake header	2019-04-09 17:08:15 -05:00