Merge pull request #125 from vulcanize/decode-packed-addresses

(VDB-768) Allow addresses in packed slots to be decoded

.travis.yml

@@ -1,27 +1,24 @@
 dist: trusty
 language: go
 go:
-  - 1.11
+  - 1.12
 services:
   - postgresql
 addons:
-  postgresql: '9.6'
+  postgresql: '11.2'
 go_import_path: github.com/vulcanize/vulcanizedb
 before_install:
   - make installtools
-  - bash ./scripts/install-postgres-10.sh
+  - bash ./scripts/install-postgres-11.sh
   - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
   - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
   - sudo apt-get update && sudo apt-get install yarn
 before_script:
   - sudo -u postgres createdb vulcanize_private
   - make migrate NAME=vulcanize_private
-  - cd postgraphile && yarn
 script:
-  - yarn test
+  - env GO111MODULE=on make test
-  - cd ../
+  - env GO111MODULE=on make integrationtest
-  - make test
-  - make integrationtest
 notifications:
   email: false
 env:

Gopkg.lock

@@ -1,563 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:48a213e9dc4880bbbd6999309a476fa4d3cc67560aa7127154cf8ea95bd464c2"
-  name = "github.com/allegro/bigcache"
-  packages = [
-    ".",
-    "queue",
-  ]
-  pruneopts = ""
-  revision = "f31987a23e44c5121ef8c8b2f2ea2e8ffa37b068"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
-  name = "github.com/aristanetworks/goarista"
-  packages = ["monotime"]
-  pruneopts = ""
-  revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
-  name = "github.com/btcsuite/btcd"
-  packages = ["btcec"]
-  pruneopts = ""
-  revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
-
-[[projects]]
-  digest = "1:5d47691333460db6ac83ced03c79b4bdb9aff3e322be24affb7855bed8affc6c"
-  name = "github.com/dave/jennifer"
-  packages = ["jen"]
-  pruneopts = ""
-  revision = "14e399b6b5e8456c66c45c955fc27b568bacb5c9"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
-  name = "github.com/deckarep/golang-set"
-  packages = ["."]
-  pruneopts = ""
-  revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
-  version = "v1.7.1"
-
-[[projects]]
-  digest = "1:90d36f5b581e95e00ced808cd48824ed6c320c25887828cce461bdef4cb7bc7c"
-  name = "github.com/ethereum/go-ethereum"
-  packages = [
-    ".",
-    "accounts",
-    "accounts/abi",
-    "accounts/abi/bind",
-    "accounts/keystore",
-    "common",
-    "common/bitutil",
-    "common/hexutil",
-    "common/math",
-    "common/mclock",
-    "common/prque",
-    "core/rawdb",
-    "core/types",
-    "crypto",
-    "crypto/ecies",
-    "crypto/secp256k1",
-    "ethclient",
-    "ethdb",
-    "ethdb/leveldb",
-    "ethdb/memorydb",
-    "event",
-    "log",
-    "metrics",
-    "p2p",
-    "p2p/discover",
-    "p2p/discv5",
-    "p2p/enode",
-    "p2p/enr",
-    "p2p/nat",
-    "p2p/netutil",
-    "params",
-    "rlp",
-    "rpc",
-    "trie",
-  ]
-  pruneopts = ""
-  revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
-
-[[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  version = "v1.4.7"
-
-[[projects]]
-  digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
-  name = "github.com/go-stack/stack"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = ""
-  revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
-
-[[projects]]
-  digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
-  name = "github.com/hashicorp/golang-lru"
-  packages = [
-    ".",
-    "simplelru",
-  ]
-  pruneopts = ""
-  revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
-
-[[projects]]
-  digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
-  name = "github.com/hashicorp/hcl"
-  packages = [
-    ".",
-    "hcl/ast",
-    "hcl/parser",
-    "hcl/printer",
-    "hcl/scanner",
-    "hcl/strconv",
-    "hcl/token",
-    "json/parser",
-    "json/scanner",
-    "json/token",
-  ]
-  pruneopts = ""
-  revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
-  name = "github.com/hpcloud/tail"
-  packages = [
-    ".",
-    "ratelimiter",
-    "util",
-    "watch",
-    "winfile",
-  ]
-  pruneopts = ""
-  revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
-  name = "github.com/huin/goupnp"
-  packages = [
-    ".",
-    "dcps/internetgateway1",
-    "dcps/internetgateway2",
-    "httpu",
-    "scpd",
-    "soap",
-    "ssdp",
-  ]
-  pruneopts = ""
-  revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
-
-[[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = ""
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
-  name = "github.com/jackpal/go-nat-pmp"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
-  name = "github.com/jmoiron/sqlx"
-  packages = [
-    ".",
-    "reflectx",
-  ]
-  pruneopts = ""
-  revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
-
-[[projects]]
-  digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3"
-  name = "github.com/konsorten/go-windows-terminal-sequences"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
-  name = "github.com/lib/pq"
-  packages = [
-    ".",
-    "oid",
-  ]
-  pruneopts = ""
-  revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
-
-[[projects]]
-  digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2353362d570a7bfa228149c62842019201cfb71"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  pruneopts = ""
-  revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
-  name = "github.com/onsi/ginkgo"
-  packages = [
-    ".",
-    "config",
-    "internal/codelocation",
-    "internal/containernode",
-    "internal/failer",
-    "internal/leafnodes",
-    "internal/remote",
-    "internal/spec",
-    "internal/spec_iterator",
-    "internal/specrunner",
-    "internal/suite",
-    "internal/testingtproxy",
-    "internal/writer",
-    "reporters",
-    "reporters/stenographer",
-    "reporters/stenographer/support/go-colorable",
-    "reporters/stenographer/support/go-isatty",
-    "types",
-  ]
-  pruneopts = ""
-  revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
-  version = "v1.6.0"
-
-[[projects]]
-  digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
-  name = "github.com/onsi/gomega"
-  packages = [
-    ".",
-    "format",
-    "ghttp",
-    "internal/assertion",
-    "internal/asyncassertion",
-    "internal/oraclematcher",
-    "internal/testingtsupport",
-    "matchers",
-    "matchers/support/goraph/bipartitegraph",
-    "matchers/support/goraph/edge",
-    "matchers/support/goraph/node",
-    "matchers/support/goraph/util",
-    "types",
-  ]
-  pruneopts = ""
-  revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
-  version = "v1.4.2"
-
-[[projects]]
-  digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
-  name = "github.com/pborman/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
-  version = "v1.2"
-
-[[projects]]
-  digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
-  name = "github.com/rjeczalik/notify"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
-  version = "v0.9.1"
-
-[[projects]]
-  digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
-  name = "github.com/rs/cors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
-  version = "v1.5.0"
-
-[[projects]]
-  digest = "1:9d57e200ef5ccc4217fe0a34287308bac652435e7c6513f6263e0493d2245c56"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = ""
-  revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
-  name = "github.com/spf13/afero"
-  packages = [
-    ".",
-    "mem",
-  ]
-  pruneopts = ""
-  revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
-  version = "v1.1.2"
-
-[[projects]]
-  digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8965335b8c7107321228e3e3702cab9832751bac"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  pruneopts = ""
-  revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = ""
-  revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
-  version = "v1.0.2"
-
-[[projects]]
-  digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
-  name = "github.com/syndtr/goleveldb"
-  packages = [
-    "leveldb",
-    "leveldb/cache",
-    "leveldb/comparer",
-    "leveldb/errors",
-    "leveldb/filter",
-    "leveldb/iterator",
-    "leveldb/journal",
-    "leveldb/memdb",
-    "leveldb/opt",
-    "leveldb/storage",
-    "leveldb/table",
-    "leveldb/util",
-  ]
-  pruneopts = ""
-  revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
-
-[[projects]]
-  branch = "master"
-  digest = "1:59b49c47c11a48f1054529207f65907c014ecf5f9a7c0d9c0f1616dec7b062ed"
-  name = "golang.org/x/crypto"
-  packages = [
-    "pbkdf2",
-    "scrypt",
-    "sha3",
-    "ssh/terminal",
-  ]
-  pruneopts = ""
-  revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
-
-[[projects]]
-  branch = "master"
-  digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "html",
-    "html/atom",
-    "html/charset",
-    "websocket",
-  ]
-  pruneopts = ""
-  revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = ""
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows",
-  ]
-  pruneopts = ""
-  revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
-
-[[projects]]
-  digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
-  name = "golang.org/x/text"
-  packages = [
-    "encoding",
-    "encoding/charmap",
-    "encoding/htmlindex",
-    "encoding/internal",
-    "encoding/internal/identifier",
-    "encoding/japanese",
-    "encoding/korean",
-    "encoding/simplifiedchinese",
-    "encoding/traditionalchinese",
-    "encoding/unicode",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "internal/utf8internal",
-    "language",
-    "runes",
-    "transform",
-    "unicode/cldr",
-    "unicode/norm",
-  ]
-  pruneopts = ""
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
-  name = "gopkg.in/fsnotify.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
-  source = "gopkg.in/fsnotify/fsnotify.v1"
-  version = "v1.4.7"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
-  name = "gopkg.in/natefinch/npipe.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
-
-[[projects]]
-  branch = "v1"
-  digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
-  name = "gopkg.in/tomb.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
-
-[[projects]]
-  digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/dave/jennifer/jen",
-    "github.com/ethereum/go-ethereum",
-    "github.com/ethereum/go-ethereum/accounts/abi",
-    "github.com/ethereum/go-ethereum/accounts/abi/bind",
-    "github.com/ethereum/go-ethereum/common",
-    "github.com/ethereum/go-ethereum/common/hexutil",
-    "github.com/ethereum/go-ethereum/core/rawdb",
-    "github.com/ethereum/go-ethereum/core/types",
-    "github.com/ethereum/go-ethereum/crypto",
-    "github.com/ethereum/go-ethereum/ethclient",
-    "github.com/ethereum/go-ethereum/ethdb",
-    "github.com/ethereum/go-ethereum/p2p",
-    "github.com/ethereum/go-ethereum/p2p/discv5",
-    "github.com/ethereum/go-ethereum/rlp",
-    "github.com/ethereum/go-ethereum/rpc",
-    "github.com/hashicorp/golang-lru",
-    "github.com/hpcloud/tail",
-    "github.com/jmoiron/sqlx",
-    "github.com/lib/pq",
-    "github.com/mitchellh/go-homedir",
-    "github.com/onsi/ginkgo",
-    "github.com/onsi/gomega",
-    "github.com/onsi/gomega/ghttp",
-    "github.com/sirupsen/logrus",
-    "github.com/spf13/cobra",
-    "github.com/spf13/viper",
-    "golang.org/x/net/context",
-    "golang.org/x/sync/errgroup",
-    "gopkg.in/tomb.v1",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,54 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[override]]
-  name = "gopkg.in/fsnotify.v1"
-  source = "gopkg.in/fsnotify/fsnotify.v1"
-
-[[override]]
-  name = "github.com/pressly/sup"
-  version = "0.5.3"
-
-[[constraint]]
-  name = "github.com/onsi/ginkgo"
-  version = "1.4.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jmoiron/sqlx"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/lib/pq"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "1.2.0"
-
-[[constraint]]
-  name = "github.com/spf13/cobra"
-  version = "0.0.1"
-
-[[constraint]]
-  name = "github.com/ethereum/go-ethereum"
-  revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"

Makefile

@@ -3,11 +3,6 @@ BASE = $(GOPATH)/src/$(PACKAGE)
 PKGS = go list ./... | grep -v "^vendor/"
 
 # Tools
-## Dependency management
-DEP = $(BIN)/dep
-$(BIN)/dep:
-	go get -u github.com/golang/dep/cmd/dep
-
 ## Testing library
 GINKGO = $(BIN)/ginkgo
 $(BIN)/ginkgo:
@@ -32,7 +27,7 @@ $(BIN)/gometalinter.v2:
 
 
 .PHONY: installtools
-installtools: | $(LINT) $(GOOSE) $(GINKGO) $(DEP)
+installtools: | $(LINT) $(GOOSE) $(GINKGO)
 	echo "Installing tools"
 
 .PHONY: metalint
@@ -58,11 +53,7 @@ integrationtest: | $(GINKGO) $(LINT)
 	go fmt ./...
 	$(GINKGO) -r integration_test/
 
-.PHONY: dep
+build:
-dep: | $(DEP)
-	$(DEP) ensure
-
-build: dep
 	go fmt ./...
 	go build
 

README.md

@@ -7,50 +7,56 @@
 
 
 ## Table of Contents
-1. [Background](../staging/README.md#background)
+1. [Background](#background)
-1. [Dependencies](../staging/README.md#dependencies)
+1. [Install](#install)
-1. [Install](../staging/README.md#install)
+1. [Usage](#usage)
-1. [Usage](../staging/README.md#usage)
+1. [Contributing](#contributing)
-1. [Tests](../staging/README.md#tests)
+1. [License](#license)
-1. [API](../staging/README.md#API)
-1. [Contributing](../staging/README.md#contributing)
-1. [License](../staging/README.md#license)
 
 
 ## Background
 The same data structures and encodings that make Ethereum an effective and trust-less distributed virtual machine
-complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by 
+complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by
-providing a suite of tools to ease the extraction and transformation of data into a more useful state.
+providing a suite of tools to ease the extraction and transformation of data into a more useful state, including
+allowing for exposing aggregate data from a suite of smart contracts.
 
+VulanizeDB includes processes that sync, transform and expose data. Syncing involves
+querying an Ethereum node and then persisting core data into a Postgres database. Transforming focuses on using previously synced data to
+query for and transform log event and storage data for specifically configured smart contract addresses. Exposing data is a matter of getting
+data from VulcanizeDB's underlying Postgres database and making it accessible.
 
-## Dependencies
+![VulcanizeDB Overview Diagram](documentation/diagrams/vdb-overview.png)
- - Go 1.11+
+
- - Postgres 10.6
+## Install
+
+1. [Dependencies](#dependencies)
+1. [Building the project](#building-the-project)
+1. [Setting up the database](#setting-up-the-database)
+1. [Configuring a synced Ethereum node](#configuring-a-synced-ethereum-node)
+
+### Dependencies
+ - Go 1.12+
+ - Postgres 11.2
  - Ethereum Node
    - [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.23+)
    - [Parity 1.8.11+](https://github.com/paritytech/parity/releases)
 
-
-## Install
-1. [Building the project](../staging/README.md#building-the-project)
-1. [Setting up the database](../staging/README.md#setting-up-the-database)
-1. [Configuring a synced Ethereum node](../staging/README.md#configuring-a-synced-ethereum-node)
-
 ### Building the project
 Download the codebase to your local `GOPATH` via:
 
 `go get github.com/vulcanize/vulcanizedb`
 
-Move to the project directory and use [golang/dep](https://github.com/golang/dep) to install the dependencies:
+Move to the project directory:
 
 `cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
 
-`dep ensure`
+Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:
-
-Once the dependencies have been successfully installed, build the executable with:
 
 `make build`
 
+If you need to use a different dependency than what is currently defined in `go.mod`, it may helpful to look into [the replace directive](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
+This instruction enables you to point at a fork or the local filesystem for dependency resolution.
+
 If you are running into issues at this stage, ensure that `GOPATH` is defined in your shell.
 If necessary, `GOPATH` can be set in `~/.bashrc` or `~/.bash_profile`, depending upon your system.
 It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
@@ -68,7 +74,8 @@ It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
 
     * See below for configuring additional environments
     
-In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
+In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from
+localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
 
 (It should be noted that trusted auth should only be enabled on systems without sensitive data in them: development and local test databases)
 
@@ -80,8 +87,9 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
         - The IPC file is called `geth.ipc`.
         - The geth IPC file path is printed to the console when you start geth.
         - The default location is:
-          - Mac: `<full home path>/Library/Ethereum`
+          - Mac: `<full home path>/Library/Ethereum/geth.ipc`
           - Linux: `<full home path>/ethereum/geth.ipc`
+        - Note: the geth.ipc file may not exist until you've started the geth process
 
       - For Parity:
         - The IPC file is called `jsonrpc.ipc`.
@@ -98,27 +106,35 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
 
 
 ## Usage
-Usage is broken up into two processes:
+As mentioned above, VulcanizeDB's processes can be split into three categories: syncing, transforming and exposing data.
 
 ### Data syncing
-To provide data for transformations, raw Ethereum data must first be synced into vDB.
+To provide data for transformations, raw Ethereum data must first be synced into VulcanizeDB.
-This is accomplished through the use of the `headerSync`, `sync`, or `coldImport` commands.
+This is accomplished through the use of the `headerSync`, `fullSync`, or `coldImport` commands.
-These commands are described in detail [here](../staging/documentation/sync.md).
+These commands are described in detail [here](documentation/data-syncing.md).
 
 ### Data transformation
-Contract watchers use the raw data that has been synced into Postgres to filter out and apply transformations to specific data of interest.
+Data transformation uses the raw data that has been synced into Postgres to filter out and apply transformations to
+specific data of interest. Since there are different types of data that may be useful for observing smart contracts, it
+follows that there are different ways to transform this data. We've started by categorizing this into Generic and
+Custom transformers:
 
-There is a built-in `contractWatcher` command which provides generic transformation of most contract data.
+- Generic Contract Transformer: Generic contract transformation can be done using a built-in command,
-The `contractWatcher` command is described further [here](../staging/documentation/contractWatcher.md).
+`contractWatcher`, which transforms contract events provided the contract's ABI is available. It also
+provides some state variable coverage by automating polling of public methods, with some restrictions.
+`contractWatcher` is described further [here](documentation/generic-transformer.md).
 
-In many cases a custom transformer or set of transformers will need to be written to provide complete or more comprehensive coverage or to optimize other aspects of the output for a specific end-use.
+- Custom Transformers: In many cases custom transformers will need to be written to provide
-In this case we have provided the `compose`, `execute`, and `composeAndExecute` commands for running custom transformers from external repositories.
+more comprehensive coverage of contract data. In this case we have provided the `compose`, `execute`, and
+`composeAndExecute` commands for running custom transformers from external repositories. Documentation on how to write,
+build and run custom transformers as Go plugins can be found
+[here](documentation/custom-transformers.md).
 
-Usage of the `compose`, `execute`, and `composeAndExecute` commands is described further [here](../staging/documentation/composeAndExecute.md).
+### Exposing the data
+[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](documentation/postgraphile.md).
 
-Documentation on how to build custom transformers to work with these commands can be found [here](../staging/documentation/transformers.md).
 
-## Tests
+### Tests
 - Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url)
     - Note: integration tests require configuration with an archival node
 - `createdb vulcanize_private` will create the test db
@@ -126,15 +142,13 @@ Documentation on how to build custom transformers to work with these commands ca
 - `make test` will run the unit tests and skip the integration tests
 - `make integrationtest` will run just the integration tests
 
-## API
-[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](../staging/postgraphile/README.md).
-
 
 ## Contributing
-Contributions are welcome! For more on this, please see [here](../staging/documentation/contributing.md).
+Contributions are welcome!
 
-Small note: If editing the Readme, please conform to the [standard-readme specification](https://github.com/RichardLitt/standard-readme).
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
 
+For more information on contributing, please see [here](documentation/contributing.md).
 
 ## License
-[AGPL-3.0](../staging/LICENSE) © Vulcanize Inc
+[AGPL-3.0](LICENSE) © Vulcanize Inc

cmd/coldImport.go

@@ -39,6 +39,8 @@ var coldImportCmd = &cobra.Command{
 
 Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		coldImport()
 	},
 }
@@ -55,7 +57,7 @@ func coldImport() {
 	ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
 	ethDB, err := ethereum.CreateDatabase(ethDBConfig)
 	if err != nil {
-		log.Fatal("Error connecting to ethereum db: ", err)
+		LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
 	}
 	mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
 	if syncAll {
@@ -63,10 +65,10 @@ func coldImport() {
 		endingBlockNumber = mostRecentBlockNumberInDb
 	}
 	if endingBlockNumber < startingBlockNumber {
-		log.Fatal("Ending block number must be greater than starting block number for cold import.")
+		LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
 	}
 	if endingBlockNumber > mostRecentBlockNumberInDb {
-		log.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
+		LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
 	}
 
 	// init pg db
@@ -76,7 +78,7 @@ func coldImport() {
 	nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
 	coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
 	if err != nil {
-		log.Fatal("Error getting node: ", err)
+		LogWithCommand.Fatal("Error getting node: ", err)
 	}
 	pgDB := utils.LoadPostgres(databaseConfig, coldNode)
 
@@ -90,6 +92,6 @@ func coldImport() {
 	coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
 	err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
 	if err != nil {
-		log.Fatal("Error executing cold import: ", err)
+		LogWithCommand.Fatal("Error executing cold import: ", err)
 	}
 }

cmd/compose.go

@@ -101,6 +101,8 @@ single config file or in separate command instances using different config files
 Specify config location when executing the command:
 ./vulcanizedb compose --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		compose()
 	},
 }
@@ -110,23 +112,25 @@ func compose() {
 	prepConfig()
 
 	// Generate code to build the plugin according to the config file
-	log.Info("generating plugin")
+	LogWithCommand.Info("generating plugin")
 	generator, err := p2.NewGenerator(genConfig, databaseConfig)
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Debug("initializing plugin generator failed")
+		LogWithCommand.Fatal(err)
 	}
 	err = generator.GenerateExporterPlugin()
 	if err != nil {
-		log.Debug("generating plugin failed")
+		LogWithCommand.Debug("generating plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Debug("getting plugin path failed")
+		LogWithCommand.Fatal(err)
 	}
 	fmt.Printf("Composed plugin %s", pluginPath)
-	log.Info("plugin .so file output to", pluginPath)
+	LogWithCommand.Info("plugin .so file output to ", pluginPath)
 }
 
 func init() {
@@ -134,38 +138,38 @@ func init() {
 }
 
 func prepConfig() {
-	log.Info("configuring plugin")
+	LogWithCommand.Info("configuring plugin")
 	names := viper.GetStringSlice("exporter.transformerNames")
 	transformers := make(map[string]config.Transformer)
 	for _, name := range names {
 		transformer := viper.GetStringMapString("exporter." + name)
 		p, pOK := transformer["path"]
 		if !pOK || p == "" {
-			log.Fatal(name, "transformer config is missing `path` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `path` value")
 		}
 		r, rOK := transformer["repository"]
 		if !rOK || r == "" {
-			log.Fatal(name, "transformer config is missing `repository` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
 		}
 		m, mOK := transformer["migrations"]
 		if !mOK || m == "" {
-			log.Fatal(name, "transformer config is missing `migrations` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
 		}
 		mr, mrOK := transformer["rank"]
 		if !mrOK || mr == "" {
-			log.Fatal(name, "transformer config is missing `rank` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
 		}
 		rank, err := strconv.ParseUint(mr, 10, 64)
 		if err != nil {
-			log.Fatal(name, "migration `rank` can't be converted to an unsigned integer")
+			LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
 		}
 		t, tOK := transformer["type"]
 		if !tOK {
-			log.Fatal(name, "transformer config is missing `type` value")
+			LogWithCommand.Fatal(name, " transformer config is missing `type` value")
 		}
 		transformerType := config.GetTransformerType(t)
 		if transformerType == config.UnknownTransformerType {
-			log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
+			LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
 		}
 
 		transformers[name] = config.Transformer{

cmd/composeAndExecute.go

@@ -105,6 +105,8 @@ single config file or in separate command instances using different config files
 Specify config location when executing the command:
 ./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		composeAndExecute()
 	},
 }
@@ -114,44 +116,44 @@ func composeAndExecute() {
 	prepConfig()
 
 	// Generate code to build the plugin according to the config file
-	log.Info("generating plugin")
+	LogWithCommand.Info("generating plugin")
 	generator, err := p2.NewGenerator(genConfig, databaseConfig)
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	err = generator.GenerateExporterPlugin()
 	if err != nil {
-		log.Debug("generating plugin failed")
+		LogWithCommand.Debug("generating plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Get the plugin path and load the plugin
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	if !genConfig.Save {
 		defer helpers.ClearFiles(pluginPath)
 	}
-	log.Info("linking plugin", pluginPath)
+	LogWithCommand.Info("linking plugin ", pluginPath)
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
-		log.Debug("linking plugin failed")
+		LogWithCommand.Debug("linking plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Load the `Exporter` symbol from the plugin
-	log.Info("loading transformers from plugin")
+	LogWithCommand.Info("loading transformers from plugin")
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
-		log.Debug("loading Exporter symbol failed")
+		LogWithCommand.Debug("loading Exporter symbol failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
-		log.Debug("plugged-in symbol not of type Exporter")
+		LogWithCommand.Debug("plugged-in symbol not of type Exporter")
 		os.Exit(1)
 	}
 
@@ -193,5 +195,5 @@ func composeAndExecute() {
 func init() {
 	rootCmd.AddCommand(composeAndExecuteCmd)
 	composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
+	composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
 }

cmd/contractWatcher.go

@@ -26,7 +26,7 @@ import (
 
 	st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
-	lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
+	ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -79,6 +79,8 @@ Requires a .toml config file:
         piping = true
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		contractWatcher()
 	},
 }
@@ -99,22 +101,22 @@ func contractWatcher() {
 	con.PrepConfig()
 	switch mode {
 	case "header":
-		t = lt.NewTransformer(con, blockChain, &db)
+		t = ht.NewTransformer(con, blockChain, &db)
 	case "full":
 		t = ft.NewTransformer(con, blockChain, &db)
 	default:
-		log.Fatal("Invalid mode")
+		LogWithCommand.Fatal("Invalid mode")
 	}
 
 	err := t.Init()
 	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
+		LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
 	}
 
 	for range ticker.C {
 		err = t.Execute()
 		if err != nil {
-			log.Error("Execution error for transformer:", t.GetConfig().Name, err)
+			LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
 		}
 	}
 }

cmd/execute.go

@@ -17,7 +17,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"plugin"
 	syn "sync"
 	"time"
@@ -64,6 +63,8 @@ must have been composed by the same version of vulcanizedb or else it will not b
 Specify config location when executing the command:
 ./vulcanizedb execute --config=./environments/config_name.toml`,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		execute()
 	},
 }
@@ -75,30 +76,29 @@ func execute() {
 	// Get the plugin path and load the plugin
 	_, pluginPath, err := genConfig.GetPluginPaths()
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	fmt.Printf("Executing plugin %s", pluginPath)
-	log.Info("linking plugin", pluginPath)
+	LogWithCommand.Info("linking plugin ", pluginPath)
 	plug, err := plugin.Open(pluginPath)
 	if err != nil {
-		log.Debug("linking plugin failed")
+		LogWithCommand.Warn("linking plugin failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Load the `Exporter` symbol from the plugin
-	log.Info("loading transformers from plugin")
+	LogWithCommand.Info("loading transformers from plugin")
 	symExporter, err := plug.Lookup("Exporter")
 	if err != nil {
-		log.Debug("loading Exporter symbol failed")
+		LogWithCommand.Warn("loading Exporter symbol failed")
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 
 	// Assert that the symbol is of type Exporter
 	exporter, ok := symExporter.(Exporter)
 	if !ok {
-		log.Debug("plugged-in symbol not of type Exporter")
+		LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
-		os.Exit(1)
 	}
 
 	// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
@@ -139,7 +139,7 @@ func execute() {
 func init() {
 	rootCmd.AddCommand(executeCmd)
 	executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
-	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
+	executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
 }
 
 type Exporter interface {
@@ -149,7 +149,7 @@ type Exporter interface {
 func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the EventTransformerInitializer set using the watcher
-	log.Info("executing event transformers")
+	LogWithCommand.Info("executing event transformers")
 	var recheck constants.TransformerExecution
 	if recheckHeadersArg {
 		recheck = constants.HeaderRecheck
@@ -166,7 +166,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
 func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the StorageTransformerInitializer set using the storage watcher
-	log.Info("executing storage transformers")
+	LogWithCommand.Info("executing storage transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {
@@ -179,7 +179,7 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
 func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
 	defer wg.Done()
 	// Execute over the ContractTransformerInitializer set using the contract watcher
-	log.Info("executing contract_watcher transformers")
+	LogWithCommand.Info("executing contract_watcher transformers")
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 	for range ticker.C {

cmd/fullSync.go

@@ -29,14 +29,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
-// syncCmd represents the sync command
+// fullSyncCmd represents the fullSync command
-var syncCmd = &cobra.Command{
+var fullSyncCmd = &cobra.Command{
-	Use:   "sync",
+	Use:   "fullSync",
 	Short: "Syncs VulcanizeDB with local ethereum node",
 	Long: `Syncs VulcanizeDB with local ethereum node. Populates
 Postgres with blocks, transactions, receipts, and logs.
 
-./vulcanizedb sync --starting-block-number 0 --config public.toml
+./vulcanizedb fullSync --starting-block-number 0 --config public.toml
 
 Expects ethereum node to be running and requires a .toml config:
 
@@ -49,38 +49,39 @@ Expects ethereum node to be running and requires a .toml config:
   ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		sync()
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
+		fullSync()
 	},
 }
 
 func init() {
-	rootCmd.AddCommand(syncCmd)
+	rootCmd.AddCommand(fullSyncCmd)
-
+	fullSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
-	syncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
 }
 
 func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
 	populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
 	if err != nil {
-		log.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
+		LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
 	}
 	missingBlocksPopulated <- populated
 }
 
-func sync() {
+func fullSync() {
 	ticker := time.NewTicker(pollingInterval)
 	defer ticker.Stop()
 
 	blockChain := getBlockChain()
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
-		log.Error("sync: Error getting last block: ", err)
+		LogWithCommand.Error("fullSync: Error getting last block: ", err)
 	}
 	if lastBlock.Int64() == 0 {
-		log.Fatal("geth initial: state sync not finished")
+		LogWithCommand.Fatal("geth initial: state sync not finished")
 	}
 	if startingBlockNumber > lastBlock.Int64() {
-		log.Fatal("sync: starting block number > current block number")
+		LogWithCommand.Fatal("fullSync: starting block number > current block number")
 	}
 
 	db := utils.LoadPostgres(databaseConfig, blockChain.Node())
@@ -94,9 +95,9 @@ func sync() {
 		case <-ticker.C:
 			window, err := validator.ValidateBlocks()
 			if err != nil {
-				log.Error("sync: error in validateBlocks: ", err)
+				LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
 			}
-			log.Info(window.GetString())
+			LogWithCommand.Debug(window.GetString())
 		case <-missingBlocksPopulated:
 			go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
 		}

cmd/headerSync.go

@@ -50,6 +50,8 @@ Expects ethereum node to be running and requires a .toml config:
   ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
 `,
 	Run: func(cmd *cobra.Command, args []string) {
+		SubCommand = cmd.CalledAs()
+		LogWithCommand = *log.WithField("SubCommand", SubCommand)
 		headerSync()
 	},
 }
@@ -64,7 +66,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
 	if err != nil {
 		// TODO Lots of possible errors in the call stack above. If errors occur, we still put
 		// 0 in the channel, triggering another round
-		log.Error("backfillAllHeaders: Error populating headers: ", err)
+		LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
 	}
 	missingBlocksPopulated <- populated
 }
@@ -86,9 +88,9 @@ func headerSync() {
 		case <-ticker.C:
 			window, err := validator.ValidateHeaders()
 			if err != nil {
-				log.Error("headerSync: ValidateHeaders failed: ", err)
+				LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
 			}
-			log.Info(window.GetString())
+			LogWithCommand.Debug(window.GetString())
 		case n := <-missingBlocksPopulated:
 			if n == 0 {
 				time.Sleep(3 * time.Second)
@@ -101,12 +103,12 @@ func headerSync() {
 func validateArgs(blockChain *geth.BlockChain) {
 	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
-		log.Error("validateArgs: Error getting last block: ", err)
+		LogWithCommand.Error("validateArgs: Error getting last block: ", err)
 	}
 	if lastBlock.Int64() == 0 {
-		log.Fatal("geth initial: state sync not finished")
+		LogWithCommand.Fatal("geth initial: state sync not finished")
 	}
 	if startingBlockNumber > lastBlock.Int64() {
-		log.Fatal("starting block number > current block number")
+		LogWithCommand.Fatal("starting block number > current block number")
 	}
 }

cmd/root.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -47,6 +46,8 @@ var (
 	syncAll              bool
 	endingBlockNumber    int64
 	recheckHeadersArg    bool
+	SubCommand           string
+	LogWithCommand       log.Entry
 )
 
 const (
@@ -56,18 +57,26 @@ const (
 
 var rootCmd = &cobra.Command{
 	Use:              "vulcanizedb",
-	PersistentPreRun: database,
+	PersistentPreRun: initFuncs,
 }
 
 func Execute() {
 	log.Info("----- Starting vDB -----")
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
-		os.Exit(1)
 	}
 }
 
-func database(cmd *cobra.Command, args []string) {
+func initFuncs(cmd *cobra.Command, args []string) {
+	setViperConfigs()
+	logLvlErr := logLevel()
+	if logLvlErr != nil {
+		log.Fatal("Could not set log level: ", logLvlErr)
+	}
+
+}
+
+func setViperConfigs() {
 	ipc = viper.GetString("client.ipcpath")
 	levelDbPath = viper.GetString("client.leveldbpath")
 	storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
@@ -81,6 +90,19 @@ func database(cmd *cobra.Command, args []string) {
 	viper.Set("database.config", databaseConfig)
 }
 
+func logLevel() error {
+	lvl, err := log.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	log.SetLevel(lvl)
+	if lvl > log.InfoLevel {
+		log.SetReportCaller(true)
+	}
+	log.Info("Log level set to ", lvl.String())
+	return nil
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 	// When searching for env variables, replace dots in config keys with underscores
@@ -97,6 +119,7 @@ func init() {
 	rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
 	rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
 	rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -107,6 +130,7 @@ func init() {
 	viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
 	viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
 	viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }
 
 func initConfig() {
@@ -116,16 +140,14 @@ func initConfig() {
 		noConfigError := "No config file passed with --config flag"
 		fmt.Println("Error: ", noConfigError)
 		log.Fatal(noConfigError)
-		os.Exit(1)
 	}
 
 	if err := viper.ReadInConfig(); err == nil {
 		log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
 	} else {
 		invalidConfigError := "Couldn't read config file"
-		fmt.Println("Error: ", invalidConfigError)
+		formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error())
-		log.Fatal(invalidConfigError)
+		log.Fatal(formattedError)
-		os.Exit(1)
 	}
 }
 
@@ -133,7 +155,7 @@ func getBlockChain() *geth.BlockChain {
 	rawRpcClient, err := rpc.Dial(ipc)
 
 	if err != nil {
-		log.Fatal(err)
+		LogWithCommand.Fatal(err)
 	}
 	rpcClient := client.NewRpcClient(rawRpcClient, ipc)
 	ethClient := ethclient.NewClient(rawRpcClient)

db/migrations/00024_create_queued_storage.sql

@@ -5,7 +5,8 @@ CREATE TABLE public.queued_storage (
   block_hash    BYTEA,
   contract      BYTEA,
   storage_key   BYTEA,
-  storage_value BYTEA
+  storage_value BYTEA,
+  UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
 );
 
 -- +goose Down

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 10.6
+-- Dumped from database version 11.3
--- Dumped by pg_dump version 10.6
+-- Dumped by pg_dump version 11.3
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -12,23 +12,10 @@ SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
 SELECT pg_catalog.set_config('search_path', '', false);
 SET check_function_bodies = false;
+SET xmloption = content;
 SET client_min_messages = warning;
 SET row_security = off;
 
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
---
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
 SET default_tablespace = '';
 
 SET default_with_oids = false;
@@ -803,6 +790,14 @@ ALTER TABLE ONLY public.eth_nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: queued_storage queued_storage_block_height_block_hash_contract_storage_key_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.queued_storage
+    ADD CONSTRAINT queued_storage_block_height_block_hash_contract_storage_key_key UNIQUE (block_height, block_hash, contract, storage_key, storage_value);
+
+
 --
 -- Name: queued_storage queued_storage_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --

documentation/contributing.md

@@ -1,11 +1,50 @@
 # Contribution guidelines
 
-Contributions are welcome! In addition to core contributions, developers are encouraged to build their own custom transformers which
+Contributions are welcome! Please open an Issues or Pull Request for any changes.
-can be run together with other custom transformers using the [composeAndExeucte](../../staging/documentation/composeAndExecute.md) command.
+
+In addition to core contributions, developers are encouraged to build their own custom transformers which
+can be run together with other custom transformers using the [composeAndExecute](../../staging/documentation/custom-transformers.md) command.
+
+## Pull Requests
+- `go fmt` is run as part of `make test` and `make integrationtest`, please make sure to check in the format changes.
+- Ensure that new code is well tested, including integration testing if applicable.
+- Make sure the build is passing.
+- Update the README or any [documentation files](./) as necessary. If editing the Readme, please
+conform to the
+[standard-readme specification](https://github.com/RichardLitt/standard-readme).
+- Once a Pull Request has received two approvals it can be merged in by a core developer.
 
 ## Creating a new migration file
 1. `make new_migration NAME=add_columnA_to_table1`
     - This will create a new timestamped migration file in `db/migrations`
 1. Write the migration code in the created file, under the respective `goose` pragma
     - Goose automatically runs each migration in a transaction; don't add `BEGIN` and `COMMIT` statements.
-1. Core migrations should be committed in their `goose fix`ed form.
+1. Core migrations should be committed in their `goose fix`ed form. To do this, run `make version_migrations` which
+converts timestamped migrations to migrations versioned by an incremented integer.
+
+## Diagrams
+- Diagrams were created with [draw.io](https://www.draw.io).
+- To update a diagram:
+  1. Go to [draw.io](https://www.draw.io).
+  1. Click on *File > Open from* and choose the location of the diagram you want to update.
+  1. Once open in draw.io, you may update it.
+  1. Export the diagram to this repository's directory and commit it.
+
+
+## Generating the Changelog
+We use [github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) to generate release Changelogs. To be consistent with previous Changelogs, the following flags should be passed to the command:
+
+```
+--user vulcanize
+--project vulcanizedb
+--token {YOUR_GITHUB_TOKEN}
+--no-issues
+--usernames-as-github-logins
+--since-tag {PREVIOUS_RELEASE_TAG}
+```
+
+For more information on why your github token is needed, and how to generate it see [https://github
+.com/github-changelog-generator/github-changelog-generator#github-token](https://github.com/github-changelog-generator/github-changelog-generator#github-token).
+
+## Code of Conduct
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).

documentation/custom-transformers.md

@@ -1,44 +1,80 @@
-# composeAndExecute
+# Custom Transformers
-The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers.
+When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
-This is accomplished by generating a Go pluggin which allows the `vulcanizedb` binary to link to external transformers, so
+purpose can be leveraged.
-long as they abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
 
-Additionally, there are separate `compose` and `execute` commands to allow pre-building and linking to a pre-built .so file.
+Individual custom transformers can be composed together from any number of external repositories and executed as a
+single process using the `compose` and `execute` commands or the `composeAndExecute` command. This is accomplished by
+generating a Go plugin which allows the `vulcanizedb` binary to link to the external transformers, so long as they
+abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
 
-**NOTE:**
+## Writing custom transformers
-1. It is necessary that the .so file was built with the same exact dependencies that are present in the execution environment,
+For help with writing different types of custom transformers please see below:
-i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
-1. The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so in a different
-environment than the one it was composed in then the migrations for that plugin will first need to be manually ran against that environment's Postgres database.
 
-These commands require Go 1.11+ and use [Go plugins](https://golang.org/pkg/plugin/) which only work on Unix-based systems.
+Storage Transformers: transform data derived from contract storage tries
-There is also an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use vendored dependencies which
+   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
-imposes certain limitations on how the plugins are built.
+   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
 
-## Commands
+Event Transformers: transform data derived from Ethereum log events
-The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's `$GOPATH`,
+   * [Guide](../../staging/libraries/shared/factories/event/README.md)
-and that all of the transformer repositories for building the plugin are present at their `$GOPATH` directories.
+   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
+   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
+   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
 
-The `execute` command does not require the plugin transformer dependencies be located in their
+Contract Transformers: transform data derived from Ethereum log events and use it to poll public contract methods
-`$GOPATH` directories, instead it expects a prebuilt .so file (of the name specified in the config file)
+   * [Example 1](https://github.com/vulcanize/account_transformers)
-to be in `$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin
+   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
-db migrations to have already been ran against the database.
 
-compose:
+## Preparing custom transformers to work as part of a plugin
+To plug in an external transformer we need to:
 
-`./vulcanizedb compose --config=./environments/config_name.toml`
+1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
+or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
+or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
+2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
+[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
+or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
+3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
+    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
+    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
+    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
+    as they are [here](../../staging/db/migrations)
 
-execute:
+To update a plugin repository with changes to the core vulcanizedb repository, use your dependency manager to install the desired version of vDB.
 
-`./vulcanizedb execute --config=./environments/config_name.toml`
+## Building and Running Custom Transformers
+### Commands
+* The `compose`, `execute`, `composeAndExecute` commands require Go 1.11+ and use [Go plugins](https://golang
+.org/pkg/plugin/) which only work on Unix-based systems.
 
-composeAndExecute:
+* There is an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use of vendored
+dependencies which imposes certain limitations on how the plugins are built.
 
-`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
+* Separate `compose` and `execute` commands allow pre-building and linking to the pre-built .so file. So, if
+these are run independently, instead of using `composeAndExecute`, a couple of things need to be considered:
+    * It is necessary that the .so file was built with the same exact dependencies that are present in the execution
+    environment, i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
+    * The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so
+    in a different environment than the one it was composed in, then the database structure will need to be loaded 
+    into the environment's Postgres database. This can either be done by manually loading the plugin's schema into 
+    Postgres, or by manually running the plugin's migrations.
+     
+* The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's 
+`$GOPATH`, and that the plugin dependencies are present at their `$GOPATH` directories.
 
-## Flags
+* The `execute` command does not require the plugin transformer dependencies be located in their `$GOPATH` directories,
+instead it expects a .so file (of the name specified in the config file) to be in
+`$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin db migrations to
+ have already been ran against the database.
 
-The `compose` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
+ * Usage:
+     * compose: `./vulcanizedb compose --config=environments/config_name.toml`
+
+     * execute: `./vulcanizedb execute --config=environments/config_name.toml`
+
+     * composeAndExecute: `./vulcanizedb composeAndExecute --config=environments/config_name.toml`
+
+### Flags
+The `execute` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
 
 - `--recheck-headers`/`-r` - specifies whether to re-check headers for events after the header has already been queried for watched logs.
 Can be useful for redundancy if you suspect that your node is not always returning all desired logs on every query.
@@ -50,7 +86,7 @@ Defaults to `false`.
 Argument is expected to be a duration (integer measured in nanoseconds): e.g. `-q=10m30s` (for 10 minute, 30 second intervals).
 Defaults to `5m` (5 minutes).
 
-## Configuration
+### Configuration
 A .toml config file is specified when executing the commands.
 The config provides information for composing a set of transformers from external repositories:
 

documentation/data-syncing.md

@@ -0,0 +1,81 @@
+# Syncing commands
+These commands are used to sync raw Ethereum data into Postgres, with varying levels of data granularity.
+
+## headerSync
+Syncs block headers from a running Ethereum node into the VulcanizeDB table `headers`.
+- Queries the Ethereum node using RPC calls.
+- Validates headers from the last 15 blocks to ensure that data is up to date.
+- Useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart
+contract storage values or event logs).
+- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
+different from what we have already stored in the database, the header record will be updated.
+
+#### Usage
+- Run: `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
+- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    ipcPath  = <path to a running Ethereum node>
+```
+- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
+
+## fullSync
+Syncs blocks, transactions, receipts and logs from a running Ethereum node into VulcanizeDB tables named
+`blocks`, `uncles`, `full_sync_transactions`, `full_sync_receipts` and `logs`. 
+- Queries the Ethereum node using RPC calls.
+- Validates headers from the last 15 blocks to ensure that data is up to date.
+- Useful when you want to maintain a broad cache of what's happening on the blockchain.
+- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
+different from what we have already stored in the database, the header record will be updated.
+
+#### Usage
+- Run `./vulcanizedb fullSync --config <config.toml> --starting-block-number <block-number>`
+- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    ipcPath  = <path to a running Ethereum node>
+```
+- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
+
+*Please note, that if you are fast syncing your Ethereum node, wait for the initial sync to finish.*
+
+## coldImport
+Syncs VulcanizeDB from Geth's underlying LevelDB datastore and persists Ethereum blocks, 
+transactions, receipts and logs into VulcanizeDB tables named `blocks`, `uncles`, 
+`full_sync_transactions`, `full_sync_receipts` and `logs` respectively.
+
+#### Usage
+1. Ensure the Ethereum node you're point at is not running, and that it has synced to the desired block height.
+1. Run `./vulcanizedb coldImport --config <config.toml>`
+1. Optional flags:
+    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
+    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
+    - `--all`/`-a`: sync all missing blocks
+
+The config file can be formatted as follows, and must contain the LevelDB path.
+
+```toml
+[database]
+    name     = "vulcanize_public"
+    hostname = "localhost"
+    user     = "vulcanize"
+    password = "vulcanize"
+    port     = 5432
+
+[client]
+    leveldbpath = "/Users/user/Library/Ethereum/geth/chaindata"
+```

documentation/generic-transformer.md

@@ -1,4 +1,4 @@
-# contractWatcher
+# Generic Transformer
 The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available.
 It also provides some state variable coverage by automating polling of public methods, with some restrictions:
 1. The method must have 2 or less arguments

documentation/postgraphile.md

@@ -0,0 +1,34 @@
+# Postgraphile
+
+You can expose VulcanizeDB data via [Postgraphile](https://github.com/graphile/postgraphile).
+Check out [their documentation](https://www.graphile.org/postgraphile/) for the most up-to-date instructions on installing, running, and customizing Postgraphile.
+
+## Simple Setup
+
+As of April 30, 2019, you can run Postgraphile pointed at the default `vulcanize_public` database with the following commands:
+
+```
+npm install -g postgraphile
+postgraphile --connection postgres://localhost/vulcanize_public --schema=public,custom --disable-default-mutations --no-ignore-rbac
+```
+
+Arguments:
+- `--connection` specifies the database. The above command connects to the default `vulcanize_public` database 
+defined in [the example config](../environments/public.toml.example).
+- `--schema` defines what schema(s) to expose. The above exposes the `public` schema (for core VulcanizeDB data) as well as a `custom` schema (where `custom` is the name of a schema defined in executed transformers).
+- `--disable-default-mutations` prevents Postgraphile from exposing create, update, and delete operations on your data, which are otherwise enabled by default.
+- `--no-ignore-rbac` ensures that Postgraphile will only expose the tables, columns, fields, and query functions that the user has explicit access to.
+
+## Customizing Postgraphile
+
+By default, Postgraphile will expose queries for all tables defined in your chosen database/schema(s), including [filtering](https://www.graphile.org/postgraphile/filtering/) and [auto-discovered relations](https://www.graphile.org/postgraphile/relations/).
+
+If you'd like to expose more customized windows into your data, there are some techniques you can apply when writing migrations:
+
+- [Computed columns](https://www.graphile.org/postgraphile/computed-columns/) enable you to derive additional fields from types defined in your database.
+For example, you could write a function to expose a block header's state root over Postgraphile with a computed column - without modifying the `public.headers` table.
+- [Custom queries](https://www.graphile.org/postgraphile/custom-queries/) enable you to provide on-demand access to more complex data (e.g. the product of joining and filtering several tables' data based on a passed argument).
+For example, you could write a custom query to get the block timestamp for every transaction originating from a given address.
+- [Subscriptions](https://www.graphile.org/postgraphile/subscriptions/) enable you to publish data as it is coming into your database.
+
+The above list is not exhaustive - please see the Postgraphile documentation for a more comprehensive and up-to-date description of available features.

documentation/sync.md

@@ -1,26 +0,0 @@
-# Syncing commands
-These commands are used to sync raw Ethereum data into Postgres.
-
-## headerSync
-Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
-This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values or event logs).
-1. Start Ethereum node
-1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
-    
-## sync
-Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.
-This command is useful when you want to maintain a broad cache of what's happening on the blockchain.
-1. Start Ethereum node (**if fast syncing your Ethereum node, wait for initial sync to finish**)
-1. In a separate terminal start VulcanizeDB:
-    - `./vulcanizedb sync --config <config.toml> --starting-block-number <block-number>`
-
-## coldImport
-Sync VulcanizeDB from the LevelDB underlying a Geth node.
-1. Assure node is not running, and that it has synced to the desired block height.
-1. Start vulcanize_db
-   - `./vulcanizedb coldImport --config <config.toml>`
-1. Optional flags:
-    - `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
-    - `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
-    - `--all`/`-a`: sync all missing blocks

documentation/transformers.md

@@ -1,40 +0,0 @@
-# Custom transformers
-When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
-purpose can be leveraged.
-
-Individual transformers can be composed together from any number of external repositories and executed as a single process using
-the `compose` and `execute` commands or the `composeAndExecute` command.
-
-## Writing custom transformers
-For help with writing different types of custom transformers for the `composeAndExecute` set of commands, please see the below:
-
-Storage Transformers
-   * [Guide](../../staging/libraries/shared/factories/storage/README.md)
-   * [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
-
-Event Transformers
-   * [Guide](../../staging/libraries/shared/factories/event/README.md)
-   * [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
-   * [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
-
-Contract Transformers
-   * [Example 1](https://github.com/vulcanize/account_transformers)
-   * [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
-
-## Preparing custom transformers to work as part of a plugin
-To plug in an external transformer we need to:
-
-1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
-or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
-or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
-2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
-[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
-or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
-3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
-    * Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
-    * Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
-    * If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
-    as they are [here](../../staging/db/migrations)
-
-To update a plugin repository with changes to the core vulcanizedb repository, run `dep ensure` to update its dependencies.

go.mod

@@ -0,0 +1,65 @@
+module github.com/vulcanize/vulcanizedb
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/allegro/bigcache v1.1.0 // indirect
+	github.com/apilayer/freegeoip v3.5.0+incompatible // indirect
+	github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76 // indirect
+	github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc // indirect
+	github.com/cespare/cp v1.1.1 // indirect
+	github.com/dave/jennifer v1.3.0
+	github.com/deckarep/golang-set v1.7.1 // indirect
+	github.com/docker/docker v1.13.1 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/elastic/gosigar v0.10.4 // indirect
+	github.com/ethereum/go-ethereum v1.9.0
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/google/uuid v1.0.0 // indirect
+	github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598 // indirect
+	github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47
+	github.com/howeyc/fsnotify v0.9.0 // indirect
+	github.com/hpcloud/tail v1.0.0
+	github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/influxdata/influxdb v1.7.7 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
+	github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0
+	github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0 // indirect
+	github.com/lib/pq v1.0.0
+	github.com/mattn/go-colorable v0.1.2 // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/mattn/go-sqlite3 v1.11.0 // indirect
+	github.com/mitchellh/go-homedir v1.0.0
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/olekukonko/tablewriter v0.0.1 // indirect
+	github.com/onsi/ginkgo v1.6.0
+	github.com/onsi/gomega v1.4.2
+	github.com/oschwald/maxminddb-golang v1.3.1 // indirect
+	github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/pressly/goose v2.6.0+incompatible
+	github.com/prometheus/tsdb v0.9.1 // indirect
+	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rs/cors v1.5.0 // indirect
+	github.com/sirupsen/logrus v1.2.0
+	github.com/spf13/cobra v0.0.3
+	github.com/spf13/viper v1.2.0
+	github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect
+	github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f // indirect
+	github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2 // indirect
+	github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d // indirect
+	github.com/tyler-smith/go-bip39 v1.0.0 // indirect
+	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 // indirect
+	golang.org/x/net v0.0.0-20190603091049-60506f45cf65
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	google.golang.org/appengine v1.6.1 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+)

go.sum

@@ -0,0 +1,226 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/allegro/bigcache v1.1.0 h1:MLuIKTjdxDc+qsG2rhjsYjsHQC5LUGjIWzutg7M+W68=
+github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A=
+github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU=
+github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76 h1:64W/KrGykPTfDI9xTkZtnjZRYA5p2+c/IuGgjzeWCpI=
+github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc h1:nLRj+ULuRYb0qTAOnuayFXRnLjYXBots5CSp5zH4RqU=
+github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
+github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
+github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
+github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
+github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
+github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo=
+github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
+github.com/ethereum/go-ethereum v1.9.0 h1:9Kaf7UfDkV3aIUJlf14hI/GgEgRAUq60u4fBlb9dLWw=
+github.com/ethereum/go-ethereum v1.9.0/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
+github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
+github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
+github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598 h1:XLoCW/kXxbvPvp216Kq/c+TtwWYHy9sjeDidFcG45g0=
+github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw=
+github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
+github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324 h1:PV190X5/DzQ/tbFFG5YpT5mH6q+cHlfgqI5JuRnH9oE=
+github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb v1.7.7 h1:UvNzAPfBrKMENVbQ4mr4ccA9sW+W1Ihl0Yh1s0BiVAg=
+github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
+github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE=
+github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0 h1:S8kWZLXHpcOq3nGAvIs0oDgd4CXxkxE3hkDVRjTu7ro=
+github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I=
+github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo=
+github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
+github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.9.1 h1:IWaAmWkYlgG7/S4iw4IpAQt5Y35QaZM6/GsZ7GsjAuk=
+github.com/prometheus/tsdb v0.9.1/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
+github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
+github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
+github.com/rs/cors v1.5.0 h1:dgSHE6+ia18arGOTIYQKKGWLvEbGvmbNE6NfxhoNHUY=
+github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.2.0 h1:M4Rzxlu+RgU4pyBRKhKaVN1VeYOm8h2jgyXnAseDgCc=
+github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
+github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
+github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f h1:T7YHzO3/eqD/kv5m9+TLM4XuEAkN7NPj5pnZHqaOo/Q=
+github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
+github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2 h1:o6NMd68tuqfQ0ZFnz2d16xzFNLWxrCvqF40InOJJHSM=
+github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc=
+github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/tyler-smith/go-bip39 v1.0.0 h1:FOHg9gaQLeBBRbHE/QrTLfEiBHy5pQ/yXzf9JG5pYFM=
+github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
+github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
+github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
+gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

libraries/shared/factories/event/repository.go

@@ -17,14 +17,11 @@
 package event
 
 import (
-	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type Repository interface {
 	Create(headerID int64, models []interface{}) error
 	MarkHeaderChecked(headerID int64) error
-	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
-	RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
 	SetDB(db *postgres.DB)
 }

libraries/shared/factories/event/transformer.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -37,7 +36,7 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.Event
 	return transformer
 }
 
-func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+func (transformer Transformer) Execute(logs []types.Log, header core.Header) error {
 	transformerName := transformer.Config.TransformerName
 	config := transformer.Config
 

libraries/shared/factories/event/transformer_test.go

@@ -23,7 +23,6 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
@@ -60,14 +59,14 @@ var _ = Describe("Transformer", func() {
 	})
 
 	It("marks header checked if no logs returned", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]types.Log{}, headerOne)
 
 		Expect(err).NotTo(HaveOccurred())
 		repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
 	})
 
 	It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]types.Log{}, headerOne)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@@ -76,7 +75,7 @@ var _ = Describe("Transformer", func() {
 	})
 
 	It("does not call repository.MarkCheckedHeader when there are logs", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).NotTo(HaveOccurred())
 		repository.AssertMarkHeaderCheckedNotCalled()
@@ -85,14 +84,14 @@ var _ = Describe("Transformer", func() {
 	It("returns error if marking header checked returns err", func() {
 		repository.SetMarkHeaderCheckedError(fakes.FakeError)
 
-		err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
+		err := t.Execute([]types.Log{}, headerOne)
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
 
 	It("converts an eth log to an entity", func() {
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@@ -102,7 +101,7 @@ var _ = Describe("Transformer", func() {
 	It("returns an error if converter fails", func() {
 		converter.ToEntitiesError = fakes.FakeError
 
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -111,7 +110,7 @@ var _ = Describe("Transformer", func() {
 	It("converts an entity to a model", func() {
 		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
 
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
@@ -121,7 +120,7 @@ var _ = Describe("Transformer", func() {
 		converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
 		converter.ToModelsError = fakes.FakeError
 
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
@@ -130,7 +129,7 @@ var _ = Describe("Transformer", func() {
 	It("persists the record", func() {
 		converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
 
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).NotTo(HaveOccurred())
 		Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@@ -139,7 +138,7 @@ var _ = Describe("Transformer", func() {
 
 	It("returns error if persisting the record fails", func() {
 		repository.SetCreateError(fakes.FakeError)
-		err := t.Execute(logs, headerOne, constants.HeaderMissing)
+		err := t.Execute(logs, headerOne)
 
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))

libraries/shared/factories/storage/README.md

@@ -100,6 +100,7 @@ type Repository interface {
 A contract-specific implementation of the repository interface enables the transformer to write the decoded storage value to the appropriate table in postgres.
 
 The `Create` function is expected to recognize and persist a given storage value by the variable's name, as indicated on the row's metadata.
+Note: we advise silently discarding duplicates in `Create` - as it's possible that you may read the same diff several times, and an error will trigger the storage watcher to queue that diff for later processing.
 
 The `SetDB` function is required for the repository to connect to the database.
 

libraries/shared/factories/storage/transformer_test.go

@@ -100,4 +100,54 @@ var _ = Describe("Storage transformer", func() {
 		Expect(err).To(HaveOccurred())
 		Expect(err).To(MatchError(fakes.FakeError))
 	})
+
+	Describe("when a storage row contains more than one item packed in storage", func() {
+		var (
+			rawValue        = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
+			fakeBlockNumber = 123
+			fakeBlockHash   = "0x67890"
+			packedTypes     = make(map[int]utils.ValueType)
+		)
+		packedTypes[0] = utils.Uint48
+		packedTypes[1] = utils.Uint48
+
+		var fakeMetadata = utils.StorageValueMetadata{
+			Name:        "",
+			Keys:        nil,
+			Type:        utils.PackedSlot,
+			PackedTypes: packedTypes,
+		}
+
+		It("passes the decoded data items to the repository", func() {
+			mappings.Metadata = fakeMetadata
+			fakeRow := utils.StorageDiffRow{
+				Contract:     common.Address{},
+				BlockHash:    common.HexToHash(fakeBlockHash),
+				BlockHeight:  fakeBlockNumber,
+				StorageKey:   common.Hash{},
+				StorageValue: rawValue.Hash(),
+			}
+
+			err := t.Execute(fakeRow)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
+			Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
+			Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
+			expectedPassedValue := make(map[int]string)
+			expectedPassedValue[0] = "10800"
+			expectedPassedValue[1] = "172800"
+			Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
+		})
+
+		It("returns error if creating a row fails", func() {
+			mappings.Metadata = fakeMetadata
+			repository.CreateErr = fakes.FakeError
+
+			err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
+
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError(fakes.FakeError))
+		})
+	})
 })

libraries/shared/fetcher/storage_fetcher.go

@@ -17,9 +17,12 @@
 package fetcher
 
 import (
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
-	"strings"
 )
 
 type IStorageFetcher interface {
@@ -39,6 +42,7 @@ func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.S
 	if tailErr != nil {
 		errs <- tailErr
 	}
+	log.Debug("fetching storage diffs...")
 	for line := range t.Lines {
 		row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
 		if parseErr != nil {

libraries/shared/mocks/transformer.go

@@ -19,7 +19,6 @@ package mocks
 import (
 	"github.com/ethereum/go-ethereum/core/types"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -33,7 +32,7 @@ type MockTransformer struct {
 	config           shared_t.EventTransformerConfig
 }
 
-func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
+func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error {
 	if mh.ExecuteError != nil {
 		return mh.ExecuteError
 	}

libraries/shared/repository/repository.go

@@ -69,31 +69,6 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
 	return result, err
 }
 
-func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
-	var result []core.Header
-	var query string
-	var err error
-
-	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
-				AND headers.block_number >= $1
-				AND headers.eth_node_fingerprint = $2`
-		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
-	} else {
-		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
-				LEFT JOIN checked_headers on headers.id = header_id
-				WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
-				AND headers.block_number >= $1
-				AND headers.block_number <= $2
-				AND headers.eth_node_fingerprint = $3`
-		err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
-	}
-
-	return result, err
-}
-
 func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
 	// Query returns `[]driver.Value`, nullable polymorphic interface
 	var queryResult []driver.Value
@@ -101,7 +76,8 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
 		`SELECT column_name FROM information_schema.columns
 		WHERE table_schema = 'public'
 			AND table_name = 'checked_headers'
-			AND column_name ~ '_checked';`
+			AND column_name <> 'id'
+			AND column_name <> 'header_id';`
 
 	err := db.Select(&queryResult, columnNamesQuery)
 	if err != nil {
@@ -121,37 +97,47 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
 	return columnNames, nil
 }
 
-// Builds a SQL string that checks if any column value is 0, given the column names.
+// Builds a SQL string that checks if any column should be checked/rechecked.
 // Defaults to FALSE when no columns are provided.
 // Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
 //     [] => "FALSE"
-func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
+func CreateHeaderCheckedPredicateSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
-
-	var result bytes.Buffer
-
 	if len(boolColumns) == 0 {
 		return "FALSE"
 	}
 
-	result.WriteString("NOT (")
+	if recheckHeaders {
+		return createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns)
+	} else {
+		return createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns)
+	}
+}
+
+func createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns []string) string {
+	var result bytes.Buffer
+	result.WriteString(" (")
 
 	// Loop excluding last column name
 	for _, column := range boolColumns[:len(boolColumns)-1] {
-
+		result.WriteString(fmt.Sprintf("%v=0 OR ", column))
-		if recheckHeaders {
-			result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
-		} else {
-			result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
-		}
 	}
 
-	// No trailing "OR" for the last column name
+	result.WriteString(fmt.Sprintf("%v=0)", boolColumns[len(boolColumns)-1]))
-	if recheckHeaders {
+
-		result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
+	return result.String()
-	} else {
+}
-		result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
+
+func createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns []string) string {
+	var result bytes.Buffer
+	result.WriteString(" (")
 
+	// Loop excluding last column name
+	for _, column := range boolColumns[:len(boolColumns)-1] {
+		result.WriteString(fmt.Sprintf("%v<%s OR ", column, constants.RecheckHeaderCap))
 	}
+
+	// No trailing "OR" for the last column name
+	result.WriteString(fmt.Sprintf("%v<%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
 
 	return result.String()
 }

libraries/shared/repository/repository_test.go

@@ -36,12 +36,12 @@ import (
 )
 
 var _ = Describe("Repository", func() {
-	Describe("MarkHeaderChecked", func() {
+	var (
-		var (
+		checkedHeadersColumn string
-			checkedHeadersColumn string
+		db                   *postgres.DB
-			db                   *postgres.DB
+	)
-		)
 
+	Describe("MarkHeaderChecked", func() {
 		BeforeEach(func() {
 			db = test_config.NewTestDB(test_config.NewTestNode())
 			test_config.CleanTestDB(db)
@@ -73,11 +73,6 @@ var _ = Describe("Repository", func() {
 	})
 
 	Describe("MarkHeaderCheckedInTransaction", func() {
-		var (
-			checkedHeadersColumn string
-			db                   *postgres.DB
-		)
-
 		BeforeEach(func() {
 			db = test_config.NewTestDB(test_config.NewTestNode())
 			test_config.CleanTestDB(db)
@@ -114,16 +109,17 @@ var _ = Describe("Repository", func() {
 
 	Describe("MissingHeaders", func() {
 		var (
-			db                       *postgres.DB
 			headerRepository         datastore.HeaderRepository
 			startingBlockNumber      int64
 			endingBlockNumber        int64
 			eventSpecificBlockNumber int64
+			outOfRangeBlockNumber    int64
 			blockNumbers             []int64
 			headerIDs                []int64
 			notCheckedSQL            string
 			err                      error
 			hr                       r2.HeaderRepository
+			columnNames              []string
 		)
 
 		BeforeEach(func() {
@@ -133,14 +129,14 @@ var _ = Describe("Repository", func() {
 			hr = r2.NewHeaderRepository(db)
 			hr.AddCheckColumns(getExpectedColumnNames())
 
-			columnNames, err := shared.GetCheckedColumnNames(db)
+			columnNames, err = shared.GetCheckedColumnNames(db)
 			Expect(err).NotTo(HaveOccurred())
-			notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
+			notCheckedSQL = shared.CreateHeaderCheckedPredicateSQL(columnNames, constants.HeaderMissing)
 
 			startingBlockNumber = rand.Int63()
 			eventSpecificBlockNumber = startingBlockNumber + 1
 			endingBlockNumber = startingBlockNumber + 2
-			outOfRangeBlockNumber := endingBlockNumber + 1
+			outOfRangeBlockNumber = endingBlockNumber + 1
 
 			blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
 
@@ -157,6 +153,7 @@ var _ = Describe("Repository", func() {
 		})
 
 		It("only treats headers as checked if the event specific logs have been checked", func() {
+			//add a checked_header record, but don't mark it check for any of the columns
 			_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
 			Expect(err).NotTo(HaveOccurred())
 
@@ -191,83 +188,47 @@ var _ = Describe("Repository", func() {
 			Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
 			Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
 		})
-	})
 
-	Describe("RecheckHeaders", func() {
+		It("handles an ending block of -1 ", func() {
-		var (
+			endingBlock := int64(-1)
-			checkedHeadersColumn                                      string
+			headers, err := shared.MissingHeaders(startingBlockNumber, endingBlock, db, notCheckedSQL)
-			db                                                        *postgres.DB
-			headerOneID, headerTwoID, headerThreeID, headerFourID     int64
-			headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
-		)
 
-		BeforeEach(func() {
+			Expect(err).NotTo(HaveOccurred())
-			db = test_config.NewTestDB(test_config.NewTestNode())
+			Expect(len(headers)).To(Equal(4))
-			test_config.CleanTestDB(db)
+			Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
+			Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
+			Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
+			Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
 
-			// create header checked column
+		})
-			checkedHeadersColumn = "test_column_checked"
-			_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
-			Expect(migrateErr).NotTo(HaveOccurred())
 
-			// create headers
+		It("when a the `notCheckedSQL` argument allows for rechecks it returns headers where the checked count is less than the maximum", func() {
-			headerRepository := repositories.NewHeaderRepository(db)
+			columnName := columnNames[0]
-			headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
+			recheckedSQL := shared.CreateHeaderCheckedPredicateSQL([]string{columnName}, constants.HeaderRecheck)
-			Expect(headerOneErr).NotTo(HaveOccurred())
+			// mark every header checked at least once
-			headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
+			// header 4 is marked the maximum number of times, it it is not longer checked
-			Expect(headerTwoErr).NotTo(HaveOccurred())
-			headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
-			Expect(headerThreeErr).NotTo(HaveOccurred())
-			headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
-			Expect(headerFourErr).NotTo(HaveOccurred())
 
-			// mark every header checked at least once, with one fully rechecked (headerThree)
 			maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
 			Expect(intConversionErr).NotTo(HaveOccurred())
-			_, markHeaderOneCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerOneID, maxCheckCount)
-			Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderTwoCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerTwoID, maxCheckCount)
-			Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderThreeCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerThreeID, maxCheckCount+1)
-			Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
-			_, markHeaderFourCheckedErr := db.Exec(
-				`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
-				headerFourID, maxCheckCount)
-			Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
-		})
 
-		AfterEach(func() {
+			markHeaderOneErr := shared.MarkHeaderChecked(headerIDs[0], db, columnName)
-			_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
+			Expect(markHeaderOneErr).NotTo(HaveOccurred())
-			Expect(cleanupMigrateErr).NotTo(HaveOccurred())
+			markHeaderTwoErr := shared.MarkHeaderChecked(headerIDs[1], db, columnName)
-		})
+			Expect(markHeaderTwoErr).NotTo(HaveOccurred())
+			markHeaderThreeErr := shared.MarkHeaderChecked(headerIDs[2], db, columnName)
+			Expect(markHeaderThreeErr).NotTo(HaveOccurred())
+			for i := 0; i <= maxCheckCount; i++ {
+				markHeaderFourErr := shared.MarkHeaderChecked(headerIDs[3], db, columnName)
+				Expect(markHeaderFourErr).NotTo(HaveOccurred())
+			}
 
-		Describe("when no ending block number (ending block number == -1)", func() {
+			headers, err := shared.MissingHeaders(1, -1, db, recheckedSQL)
-			It("returns all headers since starting block where checked count is less than cap", func() {
-				headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
 
-				Expect(err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred())
-				Expect(len(headers)).To(Equal(3))
+			Expect(len(headers)).To(Equal(3))
-				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+			Expect(headers[0].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
-				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+			Expect(headers[1].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
-				Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
+			Expect(headers[2].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
-			})
-		})
-
-		Describe("when ending block number specified", func() {
-			It("returns headers between starting and ending block where checked count is less than cap", func() {
-				headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
-
-				Expect(err).NotTo(HaveOccurred())
-				Expect(len(headers)).To(Equal(2))
-				Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
-				Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
-			})
 		})
 	})
 
@@ -285,41 +246,58 @@ var _ = Describe("Repository", func() {
 		})
 	})
 
-	Describe("CreateNotCheckedSQL", func() {
+	Describe("CreateHeaderCheckedPredicateSQL", func() {
-		It("generates a correct SQL string for one column", func() {
+		Describe("for headers that haven't been checked for logs", func() {
-			columns := []string{"columnA"}
+			It("generates a correct SQL string for one column", func() {
-			expected := "NOT (columnA!=0)"
+				columns := []string{"columnA"}
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
+				expected := " (columnA=0)"
-			Expect(actual).To(Equal(expected))
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
+				Expect(actual).To(Equal(expected))
+			})
+
+			It("generates a correct SQL string for several columns", func() {
+				columns := []string{"columnA", "columnB"}
+				expected := " (columnA=0 OR columnB=0)"
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
+				Expect(actual).To(Equal(expected))
+			})
+
+			It("defaults to FALSE when there are no columns", func() {
+				expected := "FALSE"
+				actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderMissing)
+				Expect(actual).To(Equal(expected))
+			})
 		})
 
-		It("generates a correct SQL string for several columns", func() {
+		Describe("for headers that are being rechecked for logs", func() {
-			columns := []string{"columnA", "columnB"}
+			It("generates a correct SQL string for rechecking headers for one column", func() {
-			expected := "NOT (columnA!=0 AND columnB!=0)"
+				columns := []string{"columnA"}
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
+				expected := fmt.Sprintf(" (columnA<%s)", constants.RecheckHeaderCap)
-			Expect(actual).To(Equal(expected))
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
-		})
+				Expect(actual).To(Equal(expected))
+			})
 
-		It("defaults to FALSE when there are no columns", func() {
+			It("generates a correct SQL string for rechecking headers for several columns", func() {
-			expected := "FALSE"
+				columns := []string{"columnA", "columnB"}
-			actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
+				expected := fmt.Sprintf(" (columnA<%s OR columnB<%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
-			Expect(actual).To(Equal(expected))
+				actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
-		})
+				Expect(actual).To(Equal(expected))
+			})
 
-		It("generates a correct SQL string for rechecking headers", func() {
+			It("defaults to FALSE when there are no columns", func() {
-			columns := []string{"columnA", "columnB"}
+				expected := "FALSE"
-			expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
+				actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderRecheck)
-			actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
+				Expect(actual).To(Equal(expected))
-			Expect(actual).To(Equal(expected))
+			})
 		})
 	})
 })
 
 func getExpectedColumnNames() []string {
 	return []string{
-		"column_1_checked",
+		"column_1",
-		"column_2_checked",
+		"column_2",
-		"column_3_checked",
+		"column_3",
-		"column_4_checked",
+		"column_4",
 	}
 }

libraries/shared/storage/mappings.go

@@ -47,7 +47,7 @@ const (
 )
 
 func GetMapping(indexOnContract, key string) common.Hash {
-	keyBytes := common.FromHex("0x" + key + indexOnContract)
+	keyBytes := common.FromHex(key + indexOnContract)
 	encoded := crypto.Keccak256(keyBytes)
 	return common.BytesToHash(encoded)
 }

libraries/shared/storage/mappings_test.go

@@ -21,6 +21,16 @@ var _ = Describe("Mappings", func() {
 			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
 			Expect(storageKey).To(Equal(expectedStorageKey))
 		})
+
+		It("returns same result if value includes hex prefix", func() {
+			indexOfMappingOnContract := storage.IndexZero
+			keyForDesiredValueInMapping := "0x1234567890abcdef"
+
+			storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
+
+			expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
+			Expect(storageKey).To(Equal(expectedStorageKey))
+		})
 	})
 
 	Describe("GetNestedMapping", func() {

libraries/shared/storage/storage_queue.go

@@ -38,7 +38,7 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
 func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
 	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
 		block_hash, block_height, storage_key, storage_value) VALUES
-		($1, $2, $3, $4, $5)`, row.Contract.Bytes(), row.BlockHash.Bytes(),
+		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, row.Contract.Bytes(), row.BlockHash.Bytes(),
 		row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
 	return err
 }

libraries/shared/storage/storage_queue_test.go

@@ -49,11 +49,22 @@ var _ = Describe("Storage queue", func() {
 		Expect(addErr).NotTo(HaveOccurred())
 	})
 
-	It("adds a storage row to the db", func() {
+	Describe("Add", func() {
-		var result utils.StorageDiffRow
+		It("adds a storage row to the db", func() {
-		getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
+			var result utils.StorageDiffRow
-		Expect(getErr).NotTo(HaveOccurred())
+			getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
-		Expect(result).To(Equal(row))
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(result).To(Equal(row))
+		})
+
+		It("does not duplicate storage rows", func() {
+			addErr := queue.Add(row)
+			Expect(addErr).NotTo(HaveOccurred())
+			var count int
+			getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`)
+			Expect(getErr).NotTo(HaveOccurred())
+			Expect(count).To(Equal(1))
+		})
 	})
 
 	It("deletes storage row from db", func() {

libraries/shared/storage/utils/decoder.go

@@ -23,27 +23,30 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+const (
+	bitsPerByte = 8
+)
+
 func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
 	switch metadata.Type {
 	case Uint256:
-		return decodeUint256(row.StorageValue.Bytes()), nil
+		return decodeInteger(row.StorageValue.Bytes()), nil
 	case Uint48:
-		return decodeUint48(row.StorageValue.Bytes()), nil
+		return decodeInteger(row.StorageValue.Bytes()), nil
+	case Uint128:
+		return decodeInteger(row.StorageValue.Bytes()), nil
 	case Address:
 		return decodeAddress(row.StorageValue.Bytes()), nil
 	case Bytes32:
 		return row.StorageValue.Hex(), nil
+	case PackedSlot:
+		return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
 	default:
 		panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
 	}
 }
 
-func decodeUint256(raw []byte) string {
+func decodeInteger(raw []byte) string {
-	n := big.NewInt(0).SetBytes(raw)
-	return n.String()
-}
-
-func decodeUint48(raw []byte) string {
 	n := big.NewInt(0).SetBytes(raw)
 	return n.String()
 }
@@ -51,3 +54,53 @@ func decodeUint48(raw []byte) string {
 func decodeAddress(raw []byte) string {
 	return common.BytesToAddress(raw).Hex()
 }
+
+func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string {
+	storageSlotData := raw
+	decodedStorageSlotItems := map[int]string{}
+	numberOfTypes := len(packedTypes)
+
+	for position := 0; position < numberOfTypes; position++ {
+		//get length of remaining storage date
+		lengthOfStorageData := len(storageSlotData)
+
+		//get item details (type, length, starting index, value bytes)
+		itemType := packedTypes[position]
+		lengthOfItem := getNumberOfBytes(itemType)
+		itemStartingIndex := lengthOfStorageData - lengthOfItem
+		itemValueBytes := storageSlotData[itemStartingIndex:]
+
+		//decode item's bytes and set in results map
+		decodedValue := decodeIndividualItem(itemValueBytes, itemType)
+		decodedStorageSlotItems[position] = decodedValue
+
+		//pop last item off raw slot data before moving on
+		storageSlotData = storageSlotData[0:itemStartingIndex]
+	}
+
+	return decodedStorageSlotItems
+}
+
+func decodeIndividualItem(itemBytes []byte, valueType ValueType) string {
+	switch valueType {
+	case Uint48, Uint128:
+		return decodeInteger(itemBytes)
+	case Address:
+		return decodeAddress(itemBytes)
+	default:
+		panic(fmt.Sprintf("can't decode unknown type: %d", valueType))
+	}
+}
+
+func getNumberOfBytes(valueType ValueType) int {
+	switch valueType {
+	case Uint48:
+		return 48 / bitsPerByte
+	case Uint128:
+		return 128 / bitsPerByte
+	case Address:
+		return 20
+	default:
+		panic(fmt.Sprintf("ValueType %d not recognized", valueType))
+	}
+}

libraries/shared/storage/utils/decoder_test.go

@@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
 		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
 	})
 
+	It("decodes uint128", func() {
+		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
+		row := utils.StorageDiffRow{StorageValue: fakeInt}
+		metadata := utils.StorageValueMetadata{Type: utils.Uint128}
+
+		result, err := utils.Decode(row, metadata)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
+	})
+
 	It("decodes uint48", func() {
 		fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
 		row := utils.StorageDiffRow{StorageValue: fakeInt}
@@ -59,4 +70,105 @@ var _ = Describe("Storage decoder", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(result).To(Equal(fakeAddress.Hex()))
 	})
+
+	Describe("when there are multiple items packed in the storage slot", func() {
+		It("decodes uint48 items", func() {
+			//this is a real storage data example
+			packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint48
+			packedTypes[1] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+		})
+
+		It("decodes 5 uint48 items", func() {
+			//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
+			//real storage data that has several items packed into it
+			packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
+
+			packedStorage := common.HexToHash(packedStorageHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint48
+			packedTypes[1] = utils.Uint48
+			packedTypes[2] = utils.Uint48
+			packedTypes[3] = utils.Uint48
+			packedTypes[4] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+			Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String()))
+			Expect(decodedValues[3]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String()))
+			Expect(decodedValues[4]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String()))
+		})
+
+		It("decodes 2 uint128 items", func() {
+			//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
+			//real storage data that has several items packed into it
+			packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
+				"0000000000000000AB54A98CEB1F0AD2"
+			packedStorage := common.HexToHash(packedStorageHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Uint128
+			packedTypes[1] = utils.Uint128
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String()))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("38D7EA4C67FF8E502B6730000").Bytes()).String()))
+		})
+
+		It("decodes address + 2 uint48s", func() {
+			//TODO: replace with real data when available
+			addressHex := "0000000000000000000000000000000000012345"
+			packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
+			row := utils.StorageDiffRow{StorageValue: packedStorage}
+			packedTypes := map[int]utils.ValueType{}
+			packedTypes[0] = utils.Address
+			packedTypes[1] = utils.Uint48
+			packedTypes[2] = utils.Uint48
+
+			metadata := utils.StorageValueMetadata{
+				Type:        utils.PackedSlot,
+				PackedTypes: packedTypes,
+			}
+
+			result, err := utils.Decode(row, metadata)
+			decodedValues := result.(map[int]string)
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(decodedValues[0]).To(Equal("0x" + addressHex))
+			Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
+			Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
+		})
+	})
 })

libraries/shared/storage/utils/errors.go

@@ -17,12 +17,9 @@
 package utils
 
 import (
-	"errors"
 	"fmt"
 )
 
-var ErrRowExists = errors.New("parsed row for storage diff already exists")
-
 type ErrContractNotFound struct {
 	Contract string
 }

libraries/shared/storage/utils/value.go

@@ -16,27 +16,54 @@
 
 package utils
 
+import "fmt"
+
 type ValueType int
 
 const (
 	Uint256 ValueType = iota
 	Uint48
+	Uint128
 	Bytes32
 	Address
+	PackedSlot
 )
 
 type Key string
 
 type StorageValueMetadata struct {
-	Name string
+	Name        string
-	Keys map[Key]string
+	Keys        map[Key]string
-	Type ValueType
+	Type        ValueType
+	PackedNames map[int]string    //zero indexed position in map => name of packed item
+	PackedTypes map[int]ValueType //zero indexed position in map => type of packed item
 }
 
-func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata {
+func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata {
+	return getMetadata(name, keys, valueType, nil, nil)
+}
+
+func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
+	return getMetadata(name, keys, valueType, packedNames, packedTypes)
+}
+
+func getMetadata(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
+	assertPackedSlotArgs(valueType, packedNames, packedTypes)
+
 	return StorageValueMetadata{
-		Name: name,
+		Name:        name,
-		Keys: keys,
+		Keys:        keys,
-		Type: t,
+		Type:        valueType,
+		PackedNames: packedNames,
+		PackedTypes: packedTypes,
 	}
 }
+
+func assertPackedSlotArgs(valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) {
+	if valueType == PackedSlot && (packedTypes == nil || packedNames == nil) {
+		panic(fmt.Sprintf("ValueType is PackedSlot. Expected PackedNames and PackedTypes to not be nil, but got PackedNames = %v and PackedTypes = %v", packedNames, packedTypes))
+	} else if (packedNames != nil && packedTypes != nil) && valueType != PackedSlot {
+		panic(fmt.Sprintf("PackedNames and PackedTypes passed in. Expected ValueType to equal PackedSlot (%v), but got %v.", PackedSlot, valueType))
+	}
+
+}

libraries/shared/storage/utils/value_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 var _ = Describe("Storage value metadata getter", func() {
-	It("returns a storage value metadata instance with corresponding fields assigned", func() {
+	It("returns storage value metadata for a single storage variable", func() {
 		metadataName := "fake_name"
 		metadataKeys := map[utils.Key]string{"key": "value"}
 		metadataType := utils.Uint256
@@ -19,4 +19,60 @@ var _ = Describe("Storage value metadata getter", func() {
 		}
 		Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
 	})
+
+	Describe("metadata for a packed storaged slot", func() {
+		It("returns metadata for multiple storage variables", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedNames := map[int]string{0: "name"}
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			expectedMetadata := utils.StorageValueMetadata{
+				Name:        metadataName,
+				Keys:        metadataKeys,
+				Type:        metadataType,
+				PackedTypes: metadataPackedTypes,
+				PackedNames: metadataPackedNames,
+			}
+			Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata))
+		})
+
+		It("panics if PackedTypes are nil when the type is PackedSlot", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedNames := map[int]string{0: "name"}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, nil)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+
+		It("panics if PackedNames are nil when the type is PackedSlot", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.PackedSlot
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, nil, metadataPackedTypes)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+
+		It("panics if valueType is not PackedSlot if PackedNames is populated", func() {
+			metadataName := "fake_name"
+			metadataKeys := map[utils.Key]string{"key": "value"}
+			metadataType := utils.Uint48
+			metadataPackedNames := map[int]string{0: "name"}
+			metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
+
+			getMetadata := func() {
+				utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)
+			}
+			Expect(getMetadata).To(Panic())
+		})
+	})
 })

libraries/shared/transformer/event_transformer.go

@@ -20,13 +20,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type EventTransformer interface {
-	Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
+	Execute(logs []types.Log, header core.Header) error
 	GetConfig() EventTransformerConfig
 }
 

libraries/shared/watcher/event_watcher.go

@@ -96,11 +96,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
 	if err != nil {
 		return err
 	}
-	notCheckedSQL := repository.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
+	notCheckedSQL := repository.CreateHeaderCheckedPredicateSQL(checkedColumnNames, recheckHeaders)
 
 	missingHeaders, err := repository.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
 	if err != nil {
-		logrus.Error("Fetching of missing headers failed in watcher!")
+		logrus.Error("Couldn't fetch missing headers in watcher: ", err)
 		return err
 	}
 
@@ -108,7 +108,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
 		// TODO Extend FetchLogs for doing several blocks at a time
 		logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
 		if err != nil {
-			logrus.Errorf("Error while fetching logs for header %v in watcher", header.Id)
+			logrus.WithFields(logrus.Fields{
+				"headerId":    header.Id,
+				"headerHash":  header.Hash,
+				"blockNumber": header.BlockNumber,
+			}).Errorf("Couldn't fetch logs for header: %v", err)
 			return err
 		}
 
@@ -120,6 +124,7 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
 
 		transformErr := watcher.transformLogs(logs, header)
 		if transformErr != nil {
+			logrus.Error("Could not transform logs: ", transformErr)
 			return transformErr
 		}
 	}
@@ -134,7 +139,7 @@ func (watcher *EventWatcher) transformLogs(logs []types.Log, header core.Header)
 	for _, t := range watcher.Transformers {
 		transformerName := t.GetConfig().TransformerName
 		logChunk := chunkedLogs[transformerName]
-		err := t.Execute(logChunk, header, constants.HeaderMissing)
+		err := t.Execute(logChunk, header)
 		if err != nil {
 			logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
 			return err

libraries/shared/watcher/storage_watcher.go

@@ -78,7 +78,7 @@ func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
 		return
 	}
 	executeErr := storageTransformer.Execute(row)
-	if executeErr != nil && executeErr != utils.ErrRowExists {
+	if executeErr != nil {
 		logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
 		queueErr := storageWatcher.Queue.Add(row)
 		if queueErr != nil {
@@ -100,7 +100,7 @@ func (storageWatcher StorageWatcher) processQueue() {
 			continue
 		}
 		executeErr := storageTransformer.Execute(row)
-		if executeErr == nil || executeErr == utils.ErrRowExists {
+		if executeErr == nil {
 			storageWatcher.deleteRow(row.Id)
 		}
 	}

libraries/shared/watcher/storage_watcher_test.go

@@ -109,18 +109,6 @@ var _ = Describe("Storage Watcher", func() {
 				close(done)
 			})
 
-			It("does not queue row if transformer execution fails because row already exists", func(done Done) {
-				mockTransformer.ExecuteErr = utils.ErrRowExists
-
-				go storageWatcher.Execute(rows, errs, time.Hour)
-
-				Expect(<-errs).To(BeNil())
-				Consistently(func() bool {
-					return mockQueue.AddCalled
-				}).Should(BeFalse())
-				close(done)
-			})
-
 			It("queues row for later processing if transformer execution fails", func(done Done) {
 				mockTransformer.ExecuteErr = fakes.FakeError
 
@@ -199,17 +187,6 @@ var _ = Describe("Storage Watcher", func() {
 				close(done)
 			})
 
-			It("deletes row from queue if transformer execution errors because row already exists", func(done Done) {
-				mockTransformer.ExecuteErr = utils.ErrRowExists
-
-				go storageWatcher.Execute(rows, errs, time.Nanosecond)
-
-				Eventually(func() int {
-					return mockQueue.DeletePassedId
-				}).Should(Equal(row.Id))
-				close(done)
-			})
-
 			It("logs error if deleting persisted row fails", func(done Done) {
 				mockQueue.DeleteErr = fakes.FakeError
 				tempFile, fileErr := ioutil.TempFile("", "log")

main.go

@@ -3,12 +3,15 @@ package main
 import (
 	"github.com/vulcanize/vulcanizedb/cmd"
 
-	log "github.com/sirupsen/logrus"
 	"os"
+
+	log "github.com/sirupsen/logrus"
 )
 
 func main() {
-	log.SetFormatter(&log.JSONFormatter{})
+	log.SetFormatter(&log.JSONFormatter{
+		PrettyPrint: true,
+	})
 	file, err := os.OpenFile("vulcanizedb.log",
 		os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 	if err == nil {

pkg/contract_watcher/full/transformer/transformer.go

@@ -164,7 +164,7 @@ func (tr *Transformer) Init() error {
 // Iterates through stored, initialized contract objects
 // Iterates through contract's event filters, grabbing watched event logs
 // Uses converter to convert logs into custom log type
-// Persists converted logs into custuom postgres tables
+// Persists converted logs into custom postgres tables
 // Calls selected methods, using token holder address generated during event log conversion
 func (tr *Transformer) Execute() error {
 	if len(tr.Contracts) == 0 {

pkg/contract_watcher/shared/parser/parser_test.go

@@ -220,7 +220,7 @@ var _ = Describe("Parser", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			selectMethods := p.GetMethods([]string{})
-			Expect(len(selectMethods)).To(Equal(22))
+			Expect(len(selectMethods)).To(Equal(25))
 		})
 	})
 })

pkg/core/block.go

@@ -29,7 +29,7 @@ type Block struct {
 	Number       int64  `db:"number"`
 	ParentHash   string `db:"parent_hash"`
 	Size         string `db:"size"`
-	Time         int64  `db:"time"`
+	Time         uint64 `db:"time"`
 	Transactions []TransactionModel
 	UncleHash    string `db:"uncle_hash"`
 	UnclesReward string `db:"uncles_reward"`

pkg/core/blockchain.go

@@ -30,7 +30,7 @@ type BlockChain interface {
 	GetBlockByNumber(blockNumber int64) (Block, error)
 	GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error)
 	GetHeaderByNumber(blockNumber int64) (Header, error)
-	GetHeaderByNumbers(blockNumbers []int64) ([]Header, error)
+	GetHeadersByNumbers(blockNumbers []int64) ([]Header, error)
 	GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)
 	GetTransactions(transactionHashes []common.Hash) ([]TransactionModel, error)
 	LastBlock() (*big.Int, error)

pkg/core/header.go

@@ -42,7 +42,7 @@ type POAHeader struct {
 	Number      *hexutil.Big   `json:"number"           gencodec:"required"`
 	GasLimit    hexutil.Uint64 `json:"gasLimit"         gencodec:"required"`
 	GasUsed     hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
-	Time        *hexutil.Big   `json:"timestamp"        gencodec:"required"`
+	Time        hexutil.Uint64 `json:"timestamp"        gencodec:"required"`
 	Extra       hexutil.Bytes  `json:"extraData"        gencodec:"required"`
 	Hash        common.Hash    `json:"hash"`
 }

pkg/datastore/ethereum/level/database_reader.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 type Reader interface {
@@ -48,7 +49,7 @@ func (ldbr *LevelDatabaseReader) GetBlockNumber(hash common.Hash) *uint64 {
 }
 
 func (ldbr *LevelDatabaseReader) GetBlockReceipts(hash common.Hash, number uint64) types.Receipts {
-	return rawdb.ReadReceipts(ldbr.reader, hash, number)
+	return rawdb.ReadReceipts(ldbr.reader, hash, number, &params.ChainConfig{})
 }
 
 func (ldbr *LevelDatabaseReader) GetCanonicalHash(number uint64) common.Hash {

pkg/datastore/postgres/repositories/block_repository_test.go

@@ -80,7 +80,7 @@ var _ = Describe("Saving blocks", func() {
 		blockNonce := "0x881db2ca900682e9a9"
 		miner := "x123"
 		extraData := "xextraData"
-		blockTime := int64(1508981640)
+		blockTime := uint64(1508981640)
 		uncleHash := "x789"
 		blockSize := string("1000")
 		difficulty := int64(10)
@@ -98,7 +98,7 @@ var _ = Describe("Saving blocks", func() {
 			Number:       blockNumber,
 			ParentHash:   blockParentHash,
 			Size:         blockSize,
-			Time:         blockTime,
+			Time:         uint64(blockTime),
 			UncleHash:    uncleHash,
 			UnclesReward: unclesReward,
 		}

pkg/fakes/mock_blockchain.go

@@ -98,7 +98,7 @@ func (chain *MockBlockChain) GetHeaderByNumber(blockNumber int64) (core.Header,
 	return core.Header{BlockNumber: blockNumber}, nil
 }
 
-func (chain *MockBlockChain) GetHeaderByNumbers(blockNumbers []int64) ([]core.Header, error) {
+func (chain *MockBlockChain) GetHeadersByNumbers(blockNumbers []int64) ([]core.Header, error) {
 	var headers []core.Header
 	for _, blockNumber := range blockNumbers {
 		var header = core.Header{BlockNumber: int64(blockNumber)}

pkg/geth/blockchain.go

@@ -79,7 +79,7 @@ func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.
 	return blockChain.getPOWHeader(blockNumber)
 }
 
-func (blockChain *BlockChain) GetHeaderByNumbers(blockNumbers []int64) (header []core.Header, err error) {
+func (blockChain *BlockChain) GetHeadersByNumbers(blockNumbers []int64) (header []core.Header, err error) {
 	if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID {
 		return blockChain.getPOAHeaders(blockNumbers)
 	}
@@ -129,6 +129,9 @@ func (blockChain *BlockChain) GetTransactions(transactionHashes []common.Hash) (
 
 func (blockChain *BlockChain) LastBlock() (*big.Int, error) {
 	block, err := blockChain.ethClient.HeaderByNumber(context.Background(), nil)
+	if err != nil {
+		return big.NewInt(0), err
+	}
 	return block.Number, err
 }
 
@@ -159,7 +162,7 @@ func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Heade
 		Number:      POAHeader.Number.ToInt(),
 		GasLimit:    uint64(POAHeader.GasLimit),
 		GasUsed:     uint64(POAHeader.GasUsed),
-		Time:        POAHeader.Time.ToInt(),
+		Time:        uint64(POAHeader.Time),
 		Extra:       POAHeader.Extra,
 	}, POAHeader.Hash.String()), nil
 }
@@ -208,7 +211,7 @@ func (blockChain *BlockChain) getPOAHeaders(blockNumbers []int64) (headers []cor
 				Number:      POAHeader.Number.ToInt(),
 				GasLimit:    uint64(POAHeader.GasLimit),
 				GasUsed:     uint64(POAHeader.GasUsed),
-				Time:        POAHeader.Time.ToInt(),
+				Time:        uint64(POAHeader.Time),
 				Extra:       POAHeader.Extra,
 			}, POAHeader.Hash.String())
 

pkg/geth/blockchain_test.go

@@ -93,7 +93,7 @@ var _ = Describe("Geth blockchain", func() {
 			})
 
 			It("fetches headers with multiple blocks", func() {
-				_, err := blockChain.GetHeaderByNumbers([]int64{100, 99})
+				_, err := blockChain.GetHeadersByNumbers([]int64{100, 99})
 
 				Expect(err).NotTo(HaveOccurred())
 				mockRpcClient.AssertBatchCalledWith("eth_getBlockByNumber", 2)
@@ -139,7 +139,7 @@ var _ = Describe("Geth blockchain", func() {
 				blockNumber := hexutil.Big(*big.NewInt(100))
 				mockRpcClient.SetReturnPOAHeaders([]vulcCore.POAHeader{{Number: &blockNumber}})
 
-				_, err := blockChain.GetHeaderByNumbers([]int64{100, 99})
+				_, err := blockChain.GetHeadersByNumbers([]int64{100, 99})
 
 				Expect(err).NotTo(HaveOccurred())
 				mockRpcClient.AssertBatchCalledWith("eth_getBlockByNumber", 2)

pkg/geth/converters/common/block_converter.go

@@ -19,6 +19,7 @@ package common
 import (
 	"encoding/json"
 	"math/big"
+	"strconv"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -52,7 +53,7 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error)
 		Number:       gethBlock.Number().Int64(),
 		ParentHash:   gethBlock.ParentHash().Hex(),
 		Size:         gethBlock.Size().String(),
-		Time:         gethBlock.Time().Int64(),
+		Time:         gethBlock.Time(),
 		Transactions: transactions,
 		UncleHash:    gethBlock.UncleHash().Hex(),
 	}
@@ -81,7 +82,7 @@ func (bc BlockConverter) ToCoreUncle(block core.Block, uncles []*types.Header) (
 			Hash:      uncle.Hash().Hex(),
 			Raw:       raw,
 			Reward:    thisUncleReward.String(),
-			Timestamp: uncle.Time.String(),
+			Timestamp: strconv.FormatUint(uncle.Time, 10),
 		}
 		coreUncles = append(coreUncles, coreUncle)
 		totalUncleRewards.Add(totalUncleRewards, thisUncleReward)

pkg/geth/converters/common/block_converter_test.go

@@ -44,7 +44,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 		extraData, _ := hexutil.Decode("0xe4b883e5bda9e7a59ee4bb99e9b1bc")
 		nonce := types.BlockNonce{10}
 		number := int64(1)
-		time := int64(140000000)
+		time := uint64(140000000)
 
 		header := types.Header{
 			Difficulty: difficulty,
@@ -55,7 +55,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
 			Nonce:      nonce,
 			Number:     big.NewInt(number),
 			ParentHash: common.Hash{64},
-			Time:       big.NewInt(time),
+			Time:       time,
 			UncleHash:  common.Hash{128},
 		}
 		block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})

pkg/geth/converters/common/header_converter.go

@@ -18,6 +18,7 @@ package common
 
 import (
 	"encoding/json"
+	"strconv"
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -34,7 +35,7 @@ func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash str
 		Hash:        blockHash,
 		BlockNumber: gethHeader.Number.Int64(),
 		Raw:         rawHeader,
-		Timestamp:   gethHeader.Time.String(),
+		Timestamp:   strconv.FormatUint(gethHeader.Time, 10),
 	}
 	return coreHeader
 }

pkg/geth/converters/common/header_converter_test.go

@@ -19,6 +19,7 @@ package common_test
 import (
 	"encoding/json"
 	"math/big"
+	"strconv"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -37,7 +38,7 @@ var _ = Describe("Block header converter", func() {
 			ParentHash:  common.HexToHash("0xParent"),
 			ReceiptHash: common.HexToHash("0xReceipt"),
 			Root:        common.HexToHash("0xRoot"),
-			Time:        big.NewInt(123456789),
+			Time:        uint64(123456789),
 			TxHash:      common.HexToHash("0xTransaction"),
 			UncleHash:   common.HexToHash("0xUncle"),
 		}
@@ -48,7 +49,7 @@ var _ = Describe("Block header converter", func() {
 
 		Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
 		Expect(coreHeader.Hash).To(Equal(hash))
-		Expect(coreHeader.Timestamp).To(Equal(gethHeader.Time.String()))
+		Expect(coreHeader.Timestamp).To(Equal(strconv.FormatUint(gethHeader.Time, 10)))
 	})
 
 	It("includes raw bytes for header as JSON", func() {

pkg/history/populate_blocks.go

@@ -17,6 +17,7 @@
 package history
 
 import (
+	"fmt"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -35,7 +36,7 @@ func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore
 		return 0, nil
 	}
 
-	log.Printf("Backfilling %d blocks\n\n", len(blockRange))
+	log.Debug(getBlockRangeString(blockRange))
 	_, err = RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
 	if err != nil {
 		log.Error("PopulateMissingBlocks: error gettings/updating blocks: ", err)
@@ -61,3 +62,7 @@ func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datasto
 	}
 	return len(blockNumbers), nil
 }
+
+func getBlockRangeString(blockRange []int64) string {
+	return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
+}

pkg/history/populate_headers.go

@@ -24,14 +24,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
-func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
+func PopulateMissingHeaders(blockChain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
-	lastBlock, err := blockchain.LastBlock()
+	lastBlock, err := blockChain.LastBlock()
 	if err != nil {
 		log.Error("PopulateMissingHeaders: Error getting last block: ", err)
 		return 0, err
 	}
 
-	blockNumbers, err := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockchain.Node().ID)
+	blockNumbers, err := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockChain.Node().ID)
 	if err != nil {
 		log.Error("PopulateMissingHeaders: Error getting missing block numbers: ", err)
 		return 0, err
@@ -39,17 +39,17 @@ func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datasto
 		return 0, nil
 	}
 
-	log.Printf("Backfilling %d blocks\n\n", len(blockNumbers))
+	log.Debug(getBlockRangeString(blockNumbers))
-	_, err = RetrieveAndUpdateHeaders(blockchain, headerRepository, blockNumbers)
+	_, err = RetrieveAndUpdateHeaders(blockChain, headerRepository, blockNumbers)
 	if err != nil {
-		log.Error("PopulateMissingHeaders: Error getting/updating headers:", err)
+		log.Error("PopulateMissingHeaders: Error getting/updating headers: ", err)
 		return 0, err
 	}
 	return len(blockNumbers), nil
 }
 
-func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
+func RetrieveAndUpdateHeaders(blockChain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
-	headers, err := chain.GetHeaderByNumbers(blockNumbers)
+	headers, err := blockChain.GetHeadersByNumbers(blockNumbers)
 	for _, header := range headers {
 		_, err = headerRepository.CreateOrUpdateHeader(header)
 		if err != nil {

pkg/history/validation_window.go

@@ -50,6 +50,6 @@ func MakeRange(min, max int64) []int64 {
 }
 
 func (window ValidationWindow) GetString() string {
-	return fmt.Sprintf("Validating Blocks |%v|-- Validation Window --|%v}|",
+	return fmt.Sprintf("Validating Blocks |%v|-- Validation Window --|%v|",
 		window.LowerBound, window.UpperBound)
 }

pkg/plugin/manager/manager.go

@@ -17,13 +17,15 @@
 package manager
 
 import (
+	"database/sql"
 	"errors"
 	"fmt"
+	"github.com/lib/pq"
+	"github.com/pressly/goose"
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 )
 
@@ -36,6 +38,7 @@ type manager struct {
 	GenConfig config.Plugin
 	DBConfig  config.Database
 	tmpMigDir string
+	db        *sql.DB
 }
 
 // Manager requires both filled in generator and database configs
@@ -46,6 +49,22 @@ func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
 	}
 }
 
+func (m *manager) setDB() error {
+	var pgStr string
+	if len(m.DBConfig.User) > 0 && len(m.DBConfig.Password) > 0 {
+		pgStr = fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			m.DBConfig.User, m.DBConfig.Password, m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
+	} else {
+		pgStr = fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
+	}
+	dbConnector, err := pq.NewConnector(pgStr)
+	if err != nil {
+		return errors.New(fmt.Sprintf("can't connect to db: %s", err.Error()))
+	}
+	m.db = sql.OpenDB(dbConnector)
+	return nil
+}
+
 func (m *manager) RunMigrations() error {
 	// Get paths to db migrations from the plugin config
 	paths, err := m.GenConfig.GetMigrationsPaths()
@@ -73,17 +92,19 @@ func (m *manager) RunMigrations() error {
 // Setup a temporary directory to hold transformer db migrations
 func (m *manager) setupMigrationEnv() error {
 	var err error
-	m.tmpMigDir, err = helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home+".plugin_migrations"))
+	m.tmpMigDir, err = helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, ".plugin_migrations"))
 	if err != nil {
 		return err
 	}
-	err = os.RemoveAll(m.tmpMigDir)
+	removeErr := os.RemoveAll(m.tmpMigDir)
-	if err != nil {
+	if removeErr != nil {
-		return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
+		removeErrString := "unable to remove file found at %s where tmp directory needs to be written: %s"
+		return errors.New(fmt.Sprintf(removeErrString, m.tmpMigDir, removeErr.Error()))
 	}
-	err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
+	mkdirErr := os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
-	if err != nil {
+	if mkdirErr != nil {
-		return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
+		mkdirErrString := "unable to create temporary migration directory %s: %s"
+		return errors.New(fmt.Sprintf(mkdirErrString, m.tmpMigDir, mkdirErr.Error()))
 	}
 
 	return nil
@@ -120,28 +141,23 @@ func (m *manager) createMigrationCopies(paths []string) error {
 }
 
 func (m *manager) fixAndRun(path string) error {
+	// Setup DB if not set
+	if m.db == nil {
+		setErr := m.setDB()
+		if setErr != nil {
+			return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error()))
+		}
+	}
 	// Fix the migrations
-	cmd := exec.Command("goose", "fix")
+	fixErr := goose.Fix(m.tmpMigDir)
-	cmd.Dir = m.tmpMigDir
+	if fixErr != nil {
-	err := cmd.Run()
+		return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error()))
-	if err != nil {
-		return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, err.Error()))
 	}
 	// Run the copied migrations with goose
-	var pgStr string
+	upErr := goose.Up(m.db, m.tmpMigDir)
-	if len(m.DBConfig.User) > 0 && len(m.DBConfig.Password) > 0 {
+	if upErr != nil {
-		pgStr = fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+		return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, upErr.Error()))
-			m.DBConfig.User, m.DBConfig.Password, m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
-	} else {
-		pgStr = fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
 	}
-	cmd = exec.Command("goose", "postgres", pgStr, "up")
-	cmd.Dir = m.tmpMigDir
-	err = cmd.Run()
-	if err != nil {
-		return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, err.Error()))
-	}
-
 	return nil
 }
 

postgraphile/.dockerignore

@@ -1,7 +0,0 @@
-package-lock.json
-yarn.lock
-node_modules
-Dockerfile
-README.md
-spec
-.dockerignore

postgraphile/Dockerfile

@@ -1,9 +0,0 @@
-FROM mhart/alpine-node:10
-RUN apk --update --no-cache add make g++ python findutils postgresql-dev
-
-WORKDIR /app
-COPY . /app
-run yarn install
-RUN ["./node_modules/typescript/bin/tsc"]
-EXPOSE 3000
-CMD ["node", "/app/build/dist/index.js"]

postgraphile/README.md

@@ -1,51 +0,0 @@
-# Vulcanize GraphQL API
-
-This application utilizes Postgraphile to expose GraphQL endpoints for exposure of the varied data that VulcanizeDB tracks.
-
-## Docker use
-_Note: currently this image is ~500MB large (unpacked)_
-
-Build the docker image in this directory. Start the `GraphiQL` frontend by:
-* Setting the env variables for the database connection: `DATABASE_HOST`,
-  `DATABASE_NAME`, `DATABASE_USER`, `DATABASE_PASSWORD` (and optionally
-  `DATABASE_PORT` if running on non-standard port).
-  * The specified user needs to be `superuser` on the vulcanizeDB database,
-    so postgraphile can setup watch fixtures keeping track of live schema
-    changes.
-* To limit the amount of available queries in GraphQL, a restricted user can be used
-  for postgraphile introspection by adding env variables `GQ_USER` and `GQ_PASSWORD`.
-  * By doing `GRANT [SELECT | EXECUTE]` on tables/functions for this user,
-    you can selectively assign things you want available in GraphQL.
-  * You still need to pass in a superuser with `DATABASE_USER` & `DATABASE_PASSWORD` for
-    the postgraphile watch fixtures to work.
-* By default, postgraphile publishes the `public` schema. This can be expanded with for example `GQ_SCHEMAS=public,maker`
-* Run the container (ex. `docker run -e DATABASE_HOST=localhost -e DATABASE_NAME=my_database -e DATABASE_USER=superuser -e DATABASE_PASSWORD=superuser -e GQ_USER=graphql -e GQ_PASSWORD=graphql -e GQ_SCHEMAS=public,anotherSchema -d my-postgraphile-image`)
-* GraphiQL frontend is available at `:3000/graphiql`
-  GraphQL endpoint is available at `:3000/graphql`
-
-By default, this build will expose only the "public" schema and will disable mutations - to change mutation behaviour, you can use an optional config file `config.toml` and set the env var `POSTGRAPHILE_CONFIG_PATH` to point to its location. Example `toml`:
-
-```
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port = 5432
-    gq_schemas = ["public", "yourschema"]
-    gq_user = "graphql"
-    gq_password = "graphql"
-    disable_default_mutations = false
-```
-
-## Building
-
-*This application assumes the use of the [Yarn package manager](https://yarnpkg.com/en/). The use of npm may produce unexpected results.*
-
-Install dependencies with `yarn` and execute `yarn build`. The bundle produced by Webpack will be present in `build/dist/`.
-
-## Running
-
-Provide the built bundle to node as a runnable script: `node ./build/dist/vulcanize-postgraphile-server.js`
-
-## Testing
-
-Tests are executed via Jasmine with a console reporter via the `yarn test` task.

postgraphile/package.json

@@ -1,57 +0,0 @@
-{
-  "name": "vulcanizedb",
-  "version": "1.0.0",
-  "description": "",
-  "scripts": {
-    "build": "rm -rf ./build/dist && webpack --config=./webpack.config.js",
-    "lint": "tslint --project ./tsconfig.json --config ./tslint.json",
-    "start": "npm run build && node ./build/dist/vulcanize-postgraphile-server.js",
-    "dev": "./node_modules/typescript/bin/tsc && node build/dist/src/index.js",
-    "test": "rm -rf ./build/spec && tsc --build ./tsconfig.test.json && jasmine --config=./spec/support/jasmine.json",
-    "test:ci": "npm run lint && npm run test"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/vulcanize/vulcanizedb.git"
-  },
-  "author": "Vulcanize, Inc.",
-  "license": "AGPL-3.0",
-  "bugs": {
-    "url": "https://github.com/vulcanize/vulcanizedb/issues"
-  },
-  "homepage": "https://github.com/vulcanize/vulcanizedb",
-  "dependencies": {
-    "express-session": "1.15.6",
-    "graphql-subscriptions": "0.5.8",
-    "lodash": ">=4.17.11",
-    "passport": "0.4.0",
-    "pg": "6.4.2",
-    "pg-native": "3.0.0",
-    "postgraphile": "4.4.0-beta.11",
-    "subscriptions-transport-ws": "0.9.14",
-    "toml": "2.3.3"
-  },
-  "devDependencies": {
-    "@types/express": "4.16.0",
-    "@types/express-session": "1.15.10",
-    "@types/graphql": "^0.13.4",
-    "@types/jasmine": "2.8.8",
-    "@types/lodash": "4.14.116",
-    "@types/node": "^10.12.21",
-    "@types/passport": "0.4.6",
-    "@graphile-contrib/pg-simplify-inflector": "3.0.0",
-    "awesome-typescript-loader": "5.2.0",
-    "jasmine": "3.2.0",
-    "jasmine-ts-console-reporter": "3.1.1",
-    "source-map-loader": "0.2.4",
-    "tslint": "5.11.0",
-    "tslint-eslint-rules": "5.4.0",
-    "typescript": "3.0.1",
-    "webpack": "4.17.1",
-    "webpack-cli": "3.1.0",
-    "webpack-dev-server": ">=3.1.11"
-  },
-  "resolutions": {
-    "pg": "6.4.2"
-  }
-}

postgraphile/spec/config/parse.spec.ts

@@ -1,79 +0,0 @@
-import { parseConfig } from '../../src/config/parse';
-
-describe('parseConfig', () => {
-  let configPath: string;
-  let tomlContents: string;
-  let parsedToml: object;
-  let readCallback: jasmine.Spy;
-  let tomlParseCallback: jasmine.Spy;
-
-  beforeEach(() => {
-    configPath = '/example/config/path.toml';
-    tomlContents = `[database]\nname = 'example_database'\n `
-      + `hostname = 'example.com'\nport = 1234`;
-
-    parsedToml = {
-      database: {
-        hostname: 'example.com',
-        name: 'example_database',
-        port: '1234',
-        user: 'user',
-        password: 'password'
-      }
-    };
-
-    readCallback = jasmine.createSpy('readCallback');
-    readCallback.and.returnValue(tomlContents);
-
-    tomlParseCallback = jasmine.createSpy('tomlParseCallback');
-    tomlParseCallback.and.returnValue(parsedToml);
-  });
-
-  it('provides the database host', () => {
-    const databaseConfig = parseConfig(
-      readCallback, tomlParseCallback, configPath);
-
-    expect(databaseConfig.host)
-      .toEqual('postgres://user:password@example.com:1234');
-  });
-
-  it('provides the database name', () => {
-    const databaseConfig = parseConfig(
-      readCallback, tomlParseCallback, configPath);
-
-    expect(databaseConfig.database).toEqual('example_database');
-  });
-
-  it('handles a missing config path', () => {
-    const failingCall = () =>
-      parseConfig(readCallback, tomlParseCallback, '');
-
-    tomlParseCallback.and.returnValue({
-      database: { hostname: 'example.com', name: 'example_database' }
-    });
-
-    expect(failingCall).toThrow();    
-  });
-
-  it('handles a missing database host', () => {
-    const failingCall = () =>
-      parseConfig(readCallback, tomlParseCallback, configPath);
-
-    tomlParseCallback.and.returnValue({
-      database: { hostname: '', name: 'example_database' }
-    });
-
-    expect(failingCall).toThrow();    
-  });
-
-  it('handles a missing database name', () => {
-    const failingCall = () =>
-      parseConfig(readCallback, tomlParseCallback, configPath);
-
-    tomlParseCallback.and.returnValue({
-      database: { hostname: 'example.com', name: '', port: '1234' }
-    });
-
-    expect(failingCall).toThrow();    
-  });
-});

postgraphile/spec/helpers/ts_console.ts

@@ -1,7 +0,0 @@
-// NOTE: This file helps Jasmine
-// comprehend TS sourcemaps by installing a reporter
-// specifically for TypeScript
-const TSConsoleReporter = require('jasmine-ts-console-reporter');
-
-jasmine.getEnv().clearReporters();
-jasmine.getEnv().addReporter(new TSConsoleReporter());

postgraphile/spec/server/config.spec.ts

@@ -1,106 +0,0 @@
-import { PassportStatic } from 'passport';
-
-import { PostgraphileMiddleware } from '../../src/adapters/postgraphile';
-import { buildServerConfig } from '../../src/server/config';
-
-import {
-  DatabaseConfig,
-  ServerConfig,
-  ServerUtilities
-} from '../../src/server/interface';
-
-describe('buildServerConfig', () => {
-  let configParser: jasmine.Spy;
-  let postgraphileMiddleware: PostgraphileMiddleware;
-  let expressSessionHandler: jasmine.Spy;
-  let passportInitializer: jasmine.Spy;
-  let passportSessionHandler: jasmine.Spy;
-  let serverConfig: ServerConfig;
-
-  let serverUtilities: ServerUtilities;
-  let databaseConfig: DatabaseConfig;
-
-  beforeEach(() => {
-    databaseConfig = {
-      host: 'example.com',
-      database: 'example_database',
-      schemas: ['public'],
-      ownerConnectionString: 'postgres://admin:admin@host',
-      disableDefaultMutations: true
-    };
-
-    postgraphileMiddleware = jasmine
-      .createSpyObj<PostgraphileMiddleware>(['call']);
-
-    serverUtilities = {
-      pluginHook: jasmine.createSpy('pluginHook'),
-      express: jasmine.createSpy('express'),
-      expressSession: jasmine.createSpy('expressSession'),
-      httpServerFactory: jasmine.createSpy('httpServerFactory'),
-      passport: jasmine.createSpyObj<PassportStatic>(['initialize', 'session']),
-      postgraphile: jasmine.createSpy('postgraphile')
-    };
-
-    const rawConfig: object = { exampleOption: 'example value' };
-
-    configParser = jasmine.createSpy('configParser');
-    configParser.and.returnValue(rawConfig);
-
-    expressSessionHandler = jasmine.createSpy('expressSessionHandler');
-    passportInitializer = jasmine.createSpy('passportInitializer');
-    passportSessionHandler = jasmine.createSpy('passportSessionHandler');
-
-    (serverUtilities.postgraphile as jasmine.Spy)
-      .and.returnValue(postgraphileMiddleware);
-    (serverUtilities.expressSession as jasmine.Spy)
-      .and.returnValue(expressSessionHandler);
-    (serverUtilities.passport.initialize as jasmine.Spy)
-      .and.returnValue(passportInitializer);
-    (serverUtilities.passport.session as jasmine.Spy)
-      .and.returnValue(passportSessionHandler);
-
-    serverConfig = buildServerConfig(
-      serverUtilities, databaseConfig, undefined);
-  });
-
-  it('provides the Postgraphile options', () => {
-    expect(serverConfig.options).not.toBeNull();
-  });
-
-  it('it adds the express session handler as the first middleware', () => {
-    expect(serverConfig.options.webSocketMiddlewares[0])
-      .toBe(expressSessionHandler);
-  });
-
-  it('it adds the passport initializer as the second middleware', () => {
-    expect(serverConfig.options.webSocketMiddlewares[1])
-    .toBe(passportInitializer);
-  });
-
-  it('it adds the passport session handler as the third middleware', () => {
-    expect(serverConfig.options.webSocketMiddlewares[2])
-    .toBe(passportSessionHandler);
-  });
-
-  it('provides the database config to Postgraphile', () => {
-    expect(serverUtilities.postgraphile).toHaveBeenCalledWith(
-      `${databaseConfig.host}/${databaseConfig.database}`,
-      databaseConfig.schemas,
-      jasmine.any(Object));
-  });
-
-  it('provides the Postgraphile middleware', () => {
-    expect(serverConfig.middleware).toBe(postgraphileMiddleware);
-  });
-
-  it('sets the default server port', () => {
-    expect(serverConfig.port).toEqual(3000);
-  });
-
-  it('sets an explicit server port', () => {
-    const serverConfigWithPort = buildServerConfig(
-      serverUtilities, databaseConfig, '1234');
-
-    expect(serverConfigWithPort.port).toEqual(1234);
-  });
-});

postgraphile/spec/server/runtime.spec.ts

@@ -1,66 +0,0 @@
-import { Express } from 'express';
-import { PassportStatic } from 'passport';
-import { Server } from 'http';
-
-import { ServerUtilities, ServerConfig } from '../../src/server/interface';
-import { bootServer } from '../../src/server/runtime';
-import { PostgraphileMiddleware } from '../../src/adapters/postgraphile';
-
-describe('bootServer', () => {
-  let serverUtilities: ServerUtilities;
-  let serverConfig: ServerConfig;
-  let mockExpressApp: Express;
-  let mockHttpServer: Server;
-
-  beforeEach(() => {
-    serverUtilities = {
-      pluginHook: jasmine.createSpy('pluginHook'),
-      express: jasmine.createSpy('express'),
-      expressSession: jasmine.createSpy('expressSession'),
-      httpServerFactory: jasmine.createSpy('httpServerFactory'),
-      passport: jasmine.createSpyObj<PassportStatic>(['initialize', 'session']),
-      postgraphile: jasmine.createSpy('postgraphile')
-    };
-
-    serverConfig = {
-      middleware: jasmine.createSpyObj<PostgraphileMiddleware>(['call']),
-      options: {
-        appendPlugins: [],
-        disableDefaultMutations: false,
-        enableCors: true,
-        exportGqlSchemaPath: '',
-        graphiql: true,
-        ignoreRBAC: false,
-        ownerConnectionString: '',
-        pluginHook: jasmine.createSpy('pluginHook'),
-        watchPg: true,
-        webSocketMiddlewares: [] },
-      port: 5678
-    };
-
-    mockExpressApp = jasmine.createSpyObj<Express>(['use']);
-    (serverUtilities.express as jasmine.Spy)
-      .and.returnValue(mockExpressApp);
-
-    mockHttpServer = jasmine.createSpyObj<Server>(['listen']);
-    (serverUtilities.httpServerFactory as jasmine.Spy)
-      .and.returnValue(mockHttpServer);
-
-    bootServer(serverUtilities, serverConfig);
-  });
-
-  it('builds a new, Node HTTP server', () => {
-    expect(serverUtilities.httpServerFactory)
-      .toHaveBeenCalledWith(mockExpressApp);
-  });
-
-  it('provides Postgraphile middleware to the Express app', () => {
-    const useSpy = mockExpressApp.use as jasmine.Spy;
-    expect(useSpy).toHaveBeenCalledWith(serverConfig.middleware);
-  });
-
-  it('instructs the server to listen on the given port', () => {
-    const listenSpy = mockHttpServer.listen as jasmine.Spy;
-    expect(listenSpy).toHaveBeenCalledWith(serverConfig.port);
-  });
-});

postgraphile/spec/support/jasmine.json

@@ -1,11 +0,0 @@
-{
-  "spec_dir": "build/spec/",
-  "spec_files": [
-    "**/*[sS]pec.js"
-  ],
-  "helpers": [
-    "helpers/*.js"
-  ],
-  "stopSpecOnExpectationFailure": false,
-  "random": true
-}

postgraphile/src/adapters/fs.ts

@@ -1,11 +0,0 @@
-import { URL } from 'url';
-
-export type ReadFileSyncCallback = (
-  path: string | number | Buffer | URL,
-  options?: { encoding?: null | undefined; flag?: string | undefined; }
-    | null
-    | undefined
-) => string | Buffer;
-
-export type TomlParseCallback
-  = (fileContents: string) => { [key: string]: { [key: string]: any } };

postgraphile/src/adapters/http.ts

@@ -1,7 +0,0 @@
-import { IncomingMessage, ServerResponse, Server } from 'http';
-
-export type RequestListenerCallback
-  = (request: IncomingMessage, response: ServerResponse) => void;
-
-export type CreateHttpServerCallback
-  = (requestListener?: RequestListenerCallback) => Server;

postgraphile/src/adapters/postgraphile.ts

@@ -1,32 +0,0 @@
-import { RequestHandler } from 'express';
-import {PluginHookFn } from 'postgraphile/build/postgraphile/pluginHook';
-import {Plugin} from 'postgraphile';
-
-// NOTE: Shape of the middleware is not
-// currently important to this application, but if a need arises,
-// any needed shape can be assigned from a custom type here. For
-// the time being, this is a named stub to provide clarity.
-export interface PostgraphileMiddleware extends RequestHandler {}
-
-export interface PostgraphileOptions {
-  appendPlugins: Plugin[];
-  disableDefaultMutations: boolean;
-  enableCors: boolean;
-  exportGqlSchemaPath: string;
-  graphiql: boolean;
-  ignoreRBAC: boolean;
-  ownerConnectionString: string;
-  pluginHook: PluginHookFn;
-  watchPg: boolean;
-  // NOTE Shape of the middlewares is not
-  // currently important to this application, but if a need arises,
-  // any needed shape can be assigned from a custom type here.
-  webSocketMiddlewares: object[];
-}
-
-export type PostgraphileInitCallback = (
-  databaseUrl: string,
-  schemas: string[],
-  options: PostgraphileOptions
-) => PostgraphileMiddleware;
-

postgraphile/src/adapters/session.ts

@@ -1,20 +0,0 @@
-import { SessionOptions } from 'express-session';
-import { Express, RequestHandler, Handler } from 'express';
-
-export type ExpressInitCallback = () => Express;
-
-export type ExpressSessionInitCallback
-  = (options?: SessionOptions) => RequestHandler ;
-
-export type PassportInitCallback = (
-  options?: { userProperty: string; } | undefined
-) => Handler;
-
-export type PassportSessionCallback = (
-  options?: { pauseStream: boolean; } | undefined
-) => Handler;
-
-export interface StaticPassportProvider {
-  initialize: PassportInitCallback;
-  session: PassportSessionCallback;
-}

postgraphile/src/config/parse.ts

@@ -1,79 +0,0 @@
-import { CONFIG_PATH_KEY } from '../server/config';
-import { DatabaseConfig } from '../server/interface';
-import { ReadFileSyncCallback, TomlParseCallback } from '../adapters/fs';
-
-export const MISSING_PATH_MESSAGE = `No path to config toml file provided, `
-  + `please check the value of ${CONFIG_PATH_KEY} in your environment`;
-
-export const MISSING_HOST_MESSAGE = 'No database host provided in config toml';
-export const MISSING_USER_MESSAGE = 'No database user & password '
-  + 'provided in config toml';
-export const MISSING_DATABASE_MESSAGE = 'No database name provided '
-  + 'in config toml';
-
-export function parseConfig(
-  readCallback: ReadFileSyncCallback,
-  tomlParseCallback: TomlParseCallback,
-  configPath?: string
-): DatabaseConfig {
-  let host = '';
-  let port = '';
-  let database = '';
-  let user = '';
-  let password = '';
-  let gqSchemas = ['public'];
-  let gqUser = '';
-  let gqPassword = '';
-  let disableDefaultMutations = true;
-
-  if (configPath) {
-    const tomlContents = readCallback(`${configPath}`).toString();
-    const parsedToml = tomlParseCallback(tomlContents);
-
-    host = parsedToml['database']['hostname'];
-    port = parsedToml['database']['port'];
-    database = parsedToml['database']['name'];
-    user = parsedToml['database']['user'];
-    password = parsedToml['database']['password'];
-    gqSchemas = parsedToml['database']['gq_schemas'];
-    gqUser = parsedToml['database']['gq_user'] || gqUser;
-    gqPassword = parsedToml['database']['gq_password'] || gqPassword;
-    disableDefaultMutations = parsedToml['database']['disable_default_mutations'] === undefined
-      ? true
-      : parsedToml['database']['disable_default_mutations'];
-  }
-
-  // Overwrite config values with env. vars if such are set
-  host = process.env.DATABASE_HOST || host;
-  port = process.env.DATABASE_PORT || port;
-  database = process.env.DATABASE_NAME || database;
-  user = process.env.DATABASE_USER || user;
-  password = process.env.DATABASE_PASSWORD || password;
-  gqSchemas = process.env.GQ_SCHEMAS
-      ? process.env.GQ_SCHEMAS.split(',')
-      : gqSchemas;
-  gqUser = process.env.GQ_USER || gqUser;
-  gqPassword = process.env.GQ_PASSWORD || gqPassword;
-
-  if (!host) {
-    throw new Error(MISSING_HOST_MESSAGE);
-  }
-
-  if (!database) {
-    throw new Error(MISSING_DATABASE_MESSAGE);
-  }
-
-  if (!user || !password) {
-    throw new Error(MISSING_USER_MESSAGE);
-  }
-
-  return {
-    host: gqUser && gqPassword
-        ? `postgres://${gqUser}:${gqPassword}@${host}:${port}`
-        : `postgres://${user}:${password}@${host}:${port}`,
-    database,
-    schemas: gqSchemas,
-    ownerConnectionString: `postgres://${user}:${password}@${host}:${port}/${database}`,
-    disableDefaultMutations
-  };
-}

postgraphile/src/index.ts

@@ -1,38 +0,0 @@
-import { createServer } from 'http';
-import { postgraphile, makePluginHook } from 'postgraphile';
-import { readFileSync } from 'fs';
-
-import express = require('express');
-import passport = require('passport');
-import session = require('express-session');
-import toml = require('toml');
-
-const pluginHook = makePluginHook([]);
-
-import { ServerUtilities } from './server/interface';
-import { bootServer } from './server/runtime';
-import { parseConfig } from './config/parse';
-
-import {
-  buildServerConfig,
-  CONFIG_PATH_KEY,
-  SERVER_PORT_KEY
-} from './server/config';
-
-const configPath = process.env[CONFIG_PATH_KEY];
-const serverPort = process.env[SERVER_PORT_KEY];
-
-const serverUtilities: ServerUtilities = {
-  express,
-  expressSession: session,
-  httpServerFactory: createServer,
-  passport,
-  postgraphile,
-  pluginHook
-};
-
-const databaseConfig = parseConfig(readFileSync, toml.parse, configPath);
-const serverConfig = buildServerConfig(
-  serverUtilities, databaseConfig, serverPort);
-
-bootServer(serverUtilities, serverConfig);

postgraphile/src/server/config.ts

@@ -1,52 +0,0 @@
-import { ServerUtilities, DatabaseConfig, ServerConfig } from './interface';
-
-import {
-  PostgraphileMiddleware,
-  PostgraphileOptions
-} from '../adapters/postgraphile';
-
-export const CONFIG_PATH_KEY = 'POSTGRAPHILE_CONFIG_PATH';
-export const SERVER_PORT_KEY = 'SERVER_PORT';
-
-const DEFAULT_SERVER_PORT = '3000';
-
-export function buildServerConfig(
-  utilities: ServerUtilities,
-  databaseConfig: DatabaseConfig,
-  port?: string
-): ServerConfig {
-  if (!port || port.length < 1) {
-    port = DEFAULT_SERVER_PORT;
-  }
-
-  const expressSessionHandler = utilities.expressSession();
-  const passportInitializer = utilities.passport.initialize();
-  const passportSessionHandler = utilities.passport.session();
-  const pluginHook = utilities.pluginHook;
-  const PgSimplifyInflectorPlugin = require('@graphile-contrib/pg-simplify-inflector');
-
-  const options: PostgraphileOptions = {
-    appendPlugins: [PgSimplifyInflectorPlugin],
-    disableDefaultMutations: databaseConfig.disableDefaultMutations,
-    enableCors: true,
-    exportGqlSchemaPath: 'schema.graphql',
-    graphiql: true,
-    ignoreRBAC: false,
-    ownerConnectionString: databaseConfig.ownerConnectionString,
-    pluginHook: pluginHook,
-    watchPg: true,
-    webSocketMiddlewares: [
-      expressSessionHandler,
-      passportInitializer,
-      passportSessionHandler
-    ]
-  };
-
-  const middleware: PostgraphileMiddleware = utilities.postgraphile(
-    `${databaseConfig.host}/${databaseConfig.database}`,
-    databaseConfig.schemas,
-    options
-  );
-
-  return { middleware, options, port: parseInt(port, 10) };
-}

postgraphile/src/server/interface.ts

@@ -1,37 +0,0 @@
-import { CreateHttpServerCallback } from '../adapters/http';
-
-import {
-  ExpressInitCallback,
-  ExpressSessionInitCallback,
-  StaticPassportProvider
-} from '../adapters/session';
-
-import {
-  PostgraphileInitCallback,
-  PostgraphileMiddleware,
-  PostgraphileOptions
-} from '../adapters/postgraphile';
-import { PluginHookFn } from 'postgraphile/build/postgraphile/pluginHook';
-
-export interface DatabaseConfig {
-  host: string;
-  database: string;
-  schemas: string[];
-  ownerConnectionString: string;
-  disableDefaultMutations: boolean;
-}
-
-export interface ServerConfig {
-  middleware: PostgraphileMiddleware;
-  options: PostgraphileOptions;
-  port: number;
-}
-
-export interface ServerUtilities {
-  express: ExpressInitCallback;
-  expressSession: ExpressSessionInitCallback;
-  httpServerFactory: CreateHttpServerCallback;
-  passport: StaticPassportProvider;
-  postgraphile: PostgraphileInitCallback;
-  pluginHook: PluginHookFn;
-}

postgraphile/src/server/runtime.ts

@@ -1,13 +0,0 @@
-import { ServerUtilities, ServerConfig } from './interface';
-
-export function bootServer(
-  utilities: ServerUtilities,
-  config: ServerConfig
-): void {
-  const expressApp = utilities.express();
-  expressApp.use(config.middleware);
-
-  const httpServer = utilities.httpServerFactory(expressApp);
-  
-  httpServer.listen(config.port);
-}

postgraphile/tsconfig.json

@@ -1,54 +0,0 @@
-{
-  "compilerOptions": {
-    /* Basic Options */
-    "target": "es2017",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
-    "lib": ["esnext"],                        /* Specify library files to be included in the compilation:  */
-    // "allowJs": true,                       /* Allow javascript files to be compiled. */
-    // "checkJs": true,                       /* Report errors in .js files. */
-    "jsx": "react",                           /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
-    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
-    "sourceMap": true,                        /* Generates corresponding '.map' file. */
-    // "outFile": "./",                       /* Concatenate and emit output to single file. */
-    "outDir": "build/dist",                   /* Redirect output structure to the directory. */
-    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
-    "removeComments": true,                   /* Do not emit comments to output. */
-    // "noEmit": true,                        /* Do not emit outputs. */
-    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
-    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
-    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
-
-    /* Strict Type-Checking Options */
-    "strict": true,                           /* Enable all strict type-checking options. */
-    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
-    // "strictNullChecks": true,              /* Enable strict null checks. */
-    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
-    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
-    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
-
-    /* Additional Checks */
-    // "noUnusedLocals": true,                /* Report errors on unused locals. */
-    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
-    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
-    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
-
-    /* Module Resolution Options */
-    "moduleResolution": "node",                /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
-    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
-    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
-    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
-    // "typeRoots": [],                       /* List of folders to include type definitions from. */
-    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
-    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
-
-    /* Source Map Options */
-    // "sourceRoot": "./",                    /* Specify the location where debugger should locate TypeScript files instead of source locations. */
-    // "mapRoot": "./",                       /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
-    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
-
-    /* Experimental Options */
-    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
-    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
-  }
-}

postgraphile/tsconfig.test.json

@@ -1,54 +0,0 @@
-{
-  "compilerOptions": {
-    /* Basic Options */
-    "target": "es2017",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
-    "lib": ["esnext"],                        /* Specify library files to be included in the compilation:  */
-    // "allowJs": true,                       /* Allow javascript files to be compiled. */
-    // "checkJs": true,                       /* Report errors in .js files. */
-    "jsx": "react",                           /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
-    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
-    "sourceMap": true,                        /* Generates corresponding '.map' file. */
-    // "outFile": "./",                       /* Concatenate and emit output to single file. */
-    "outDir": "build/spec",              /* Redirect output structure to the directory. */
-    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
-    "removeComments": true,                   /* Do not emit comments to output. */
-    // "noEmit": true,                        /* Do not emit outputs. */
-    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
-    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
-    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
-
-    /* Strict Type-Checking Options */
-    "strict": true,                           /* Enable all strict type-checking options. */
-    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
-    // "strictNullChecks": true,              /* Enable strict null checks. */
-    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
-    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
-    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
-
-    /* Additional Checks */
-    // "noUnusedLocals": true,                /* Report errors on unused locals. */
-    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
-    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
-    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
-
-    /* Module Resolution Options */
-    "moduleResolution": "node",                /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
-    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
-    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
-    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
-    // "typeRoots": [],                       /* List of folders to include type definitions from. */
-    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
-    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
-
-    /* Source Map Options */
-    // "sourceRoot": "./",                    /* Specify the location where debugger should locate TypeScript files instead of source locations. */
-    // "mapRoot": "./",                       /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
-    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
-
-    /* Experimental Options */
-    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
-    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
-  }
-}

postgraphile/tslint.json

@@ -1,93 +0,0 @@
-{
-  "extends": [
-    "tslint-eslint-rules"
-  ],
-  "rules": {
-    "object-curly-spacing": true,
-    "ter-indent": [true, 2],
-    "array-bracket-spacing": ["error", "never"],
-    "class-name": true,
-    "comment-format": [
-      true,
-      "check-space"
-    ],
-    "curly": [true, "ignore-same-line"],
-    "eofline": true,
-    "forin": true,
-    "indent": [
-      true,
-      "spaces"
-    ],
-    "label-position": true,
-    "max-line-length": [
-      true,
-      80
-    ],
-    "member-access": [true, "no-public"],
-    "member-ordering": [true, { "order": "statics-first" }],
-    "no-arg": true,
-    "no-bitwise": true,
-    "no-console": [
-      true,
-      "debug",
-      "info",
-      "time",
-      "timeEnd",
-      "trace"
-    ],
-    "no-consecutive-blank-lines": true,
-    "no-construct": true,
-    "no-debugger": true,
-    "no-duplicate-variable": true,
-    "no-empty": false,
-    "no-eval": true,
-    "no-inferrable-types": true,
-    "no-shadowed-variable": false,
-    "no-string-literal": false,
-    "no-switch-case-fall-through": true,
-    "no-trailing-whitespace": false,
-    "no-unused-expression": false,
-    "no-use-before-declare": true,
-    "no-var-keyword": false,
-    "object-literal-sort-keys": false,
-    "one-line": [
-      true,
-      "check-open-brace",
-      "check-whitespace"
-    ],
-    "quotemark": [
-      true,
-      "single"
-    ],
-    "radix": true,
-    "semicolon": true,
-    "triple-equals": [
-      true,
-      "allow-null-check"
-    ],
-    "typedef-whitespace": [
-      true,
-      {
-        "call-signature": "nospace",
-        "index-signature": "nospace",
-        "parameter": "nospace",
-        "property-declaration": "nospace",
-        "variable-declaration": "nospace"
-      }
-    ],
-    "variable-name": false,
-    "whitespace": [
-      true,
-      "check-branch",
-      "check-decl",
-      "check-operator",
-      "check-separator",
-      "check-type"
-    ],
-
-    "use-host-property-decorator": false,
-    "no-access-missing-member": false,
-    "templates-use-public": false,
-    "brace-style": false
-  }
-}

postgraphile/webpack.config.js

@@ -1,20 +0,0 @@
-module.exports = {
-  entry: "./src/index.ts",
-  mode: "production",
-  devtool: "source-map",
-  target: 'node',
-  output: {
-    filename: "vulcanize-postgraphile-server.js",
-    path: __dirname + "/build/dist/",
-    publicPath: "build/dist/"
-  },
-  resolve: {
-    extensions: [".ts", ".tsx", ".mjs", ".js", ".json", ".css", ".png"]
-  },
-  module: {
-    rules: [
-      { test: /\.ts$/, loader: "awesome-typescript-loader" },
-      { enforce: "pre", test: /\.js$/, loader: "source-map-loader" }
-    ]
-  }
-};

scripts/install-postgres-10.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Travis doesn't support postgres 10
-# https://github.com/travis-ci/travis-ci/issues/8537
-
-set -ex
-
-echo "Installing Postgres 10"
-sudo service postgresql stop
-sudo apt-get remove -q 'postgresql-*'
-sudo apt-get update -q
-sudo apt-get install -q postgresql-10 postgresql-client-10
-sudo cp /etc/postgresql/{9.6,10}/main/pg_hba.conf
-
-echo "Restarting Postgres 10"
-sudo service postgresql restart
-
-sudo psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres

scripts/install-postgres-11.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -ex
+
+echo "Installing Postgres 11"
+sudo service postgresql stop
+sudo apt-get remove -q 'postgresql-*'
+sudo apt-get update -q
+sudo apt-get install -q postgresql-11 postgresql-client-11
+sudo cp /etc/postgresql/{9.6,11}/main/pg_hba.conf
+
+echo "Restarting Postgres 11"
+sudo service postgresql restart
+
+sudo psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres

utils/utils.go

@@ -31,7 +31,7 @@ import (
 func LoadPostgres(database config.Database, node core.Node) postgres.DB {
 	db, err := postgres.NewDB(database, node)
 	if err != nil {
-		log.Fatalf("Error loading postgres\n%v", err)
+		log.Fatal("Error loading postgres: ", err)
 	}
 	return *db
 }
@@ -65,7 +65,7 @@ func GetAbi(abiFilepath string, contractHash string, network string) string {
 	}
 	_, err := geth.ParseAbi(contractAbiString)
 	if err != nil {
-		log.Fatalln("Invalid ABI")
+		log.Fatalln("Invalid ABI: ", err)
 	}
 	return contractAbiString
 }

vendor/github.com/allegro/bigcache/.gitignore

@@ -1,5 +0,0 @@
-.idea
-.DS_Store
-/server/server.exe
-/server/server
-CHANGELOG.md

vendor/github.com/allegro/bigcache/.travis.yml

@@ -1,31 +0,0 @@
-language: go
-
-go:
-  - 1.x
-  - tip
-
-matrix:
-  allow_failures:
-    - go: tip
-  fast_finish: true
-
-before_install:
-  - go get github.com/modocache/gover
-  - go get github.com/mattn/goveralls
-  - go get golang.org/x/tools/cmd/cover
-  - go get golang.org/x/tools/cmd/goimports
-  - go get github.com/golang/lint/golint
-  - go get github.com/stretchr/testify/assert
-  - go get github.com/gordonklaus/ineffassign
-  
-script:
-  - gofiles=$(find ./ -name '*.go') && [ -z "$gofiles" ] || unformatted=$(goimports -l $gofiles) && [ -z "$unformatted" ] || (echo >&2 "Go files must be formatted with gofmt. Following files has problem:\n $unformatted" && false)
-  - diff <(echo -n) <(gofmt -s -d .)
-  - golint ./... # This won't break the build, just show warnings
-  - ineffassign .
-  - go vet ./...
-  - go test -race -count=1 -coverprofile=queue.coverprofile ./queue
-  - go test -race -count=1 -coverprofile=server.coverprofile ./server
-  - go test -race -count=1 -coverprofile=main.coverprofile
-  - $HOME/gopath/bin/gover
-  - $HOME/gopath/bin/goveralls -coverprofile=gover.coverprofile -service travis-ci

vendor/github.com/allegro/bigcache/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

vendor/github.com/allegro/bigcache/README.md

@@ -1,145 +0,0 @@
-# BigCache [![Build Status](https://travis-ci.org/allegro/bigcache.svg?branch=master)](https://travis-ci.org/allegro/bigcache) [![Coverage Status](https://coveralls.io/repos/github/allegro/bigcache/badge.svg?branch=master)](https://coveralls.io/github/allegro/bigcache?branch=master) [![GoDoc](https://godoc.org/github.com/allegro/bigcache?status.svg)](https://godoc.org/github.com/allegro/bigcache) [![Go Report Card](https://goreportcard.com/badge/github.com/allegro/bigcache)](https://goreportcard.com/report/github.com/allegro/bigcache)
-
-Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance.
-BigCache keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place,
-therefore entries (de)serialization in front of the cache will be needed in most use cases.
-
-## Usage
-
-### Simple initialization
-
-```go
-import "github.com/allegro/bigcache"
-
-cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute))
-
-cache.Set("my-unique-key", []byte("value"))
-
-entry, _ := cache.Get("my-unique-key")
-fmt.Println(string(entry))
-```
-
-### Custom initialization
-
-When cache load can be predicted in advance then it is better to use custom initialization because additional memory
-allocation can be avoided in that way.
-
-```go
-import (
-	"log"
-
-	"github.com/allegro/bigcache"
-)
-
-config := bigcache.Config {
-		// number of shards (must be a power of 2)
-		Shards: 1024,
-		// time after which entry can be evicted
-		LifeWindow: 10 * time.Minute,
-		// rps * lifeWindow, used only in initial memory allocation
-		MaxEntriesInWindow: 1000 * 10 * 60,
-		// max entry size in bytes, used only in initial memory allocation
-		MaxEntrySize: 500,
-		// prints information about additional memory allocation
-		Verbose: true,
-		// cache will not allocate more memory than this limit, value in MB
-		// if value is reached then the oldest entries can be overridden for the new ones
-		// 0 value means no size limit
-		HardMaxCacheSize: 8192,
-		// callback fired when the oldest entry is removed because of its
-		// expiration time or no space left for the new entry. Default value is nil which
-		// means no callback and it prevents from unwrapping the oldest entry.
-		OnRemove: nil,
-	}
-
-cache, initErr := bigcache.NewBigCache(config)
-if initErr != nil {
-	log.Fatal(initErr)
-}
-
-cache.Set("my-unique-key", []byte("value"))
-
-if entry, err := cache.Get("my-unique-key"); err == nil {
-	fmt.Println(string(entry))
-}
-```
-
-## Benchmarks
-
-Three caches were compared: bigcache, [freecache](https://github.com/coocood/freecache) and map.
-Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10.
-
-### Writes and reads
-
-```bash
-cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m
-
-BenchmarkMapSet-8                     	 2000000	       716 ns/op	     336 B/op	       3 allocs/op
-BenchmarkConcurrentMapSet-8           	 1000000	      1292 ns/op	     347 B/op	       8 allocs/op
-BenchmarkFreeCacheSet-8               	 3000000	       501 ns/op	     371 B/op	       3 allocs/op
-BenchmarkBigCacheSet-8                	 3000000	       482 ns/op	     303 B/op	       2 allocs/op
-BenchmarkMapGet-8                     	 5000000	       309 ns/op	      24 B/op	       1 allocs/op
-BenchmarkConcurrentMapGet-8           	 2000000	       659 ns/op	      24 B/op	       2 allocs/op
-BenchmarkFreeCacheGet-8               	 3000000	       541 ns/op	     152 B/op	       3 allocs/op
-BenchmarkBigCacheGet-8                	 3000000	       420 ns/op	     152 B/op	       3 allocs/op
-BenchmarkBigCacheSetParallel-8        	10000000	       184 ns/op	     313 B/op	       3 allocs/op
-BenchmarkFreeCacheSetParallel-8       	10000000	       195 ns/op	     357 B/op	       4 allocs/op
-BenchmarkConcurrentMapSetParallel-8   	 5000000	       242 ns/op	     200 B/op	       6 allocs/op
-BenchmarkBigCacheGetParallel-8        	20000000	       100 ns/op	     152 B/op	       4 allocs/op
-BenchmarkFreeCacheGetParallel-8       	10000000	       133 ns/op	     152 B/op	       4 allocs/op
-BenchmarkConcurrentMapGetParallel-8   	10000000	       202 ns/op	      24 B/op	       2 allocs/op
-```
-
-Writes and reads in bigcache are faster than in freecache.
-Writes to map are the slowest.
-
-### GC pause time
-
-```bash
-cd caches_bench; go run caches_gc_overhead_comparison.go
-
-Number of entries:  20000000
-GC pause for bigcache:  5.8658ms
-GC pause for freecache:  32.4341ms
-GC pause for map:  52.9661ms
-```
-
-Test shows how long are the GC pauses for caches filled with 20mln of entries.
-Bigcache and freecache have very similar GC pause time.
-It is clear that both reduce GC overhead in contrast to map
-which GC pause time took more than 10 seconds.
-
-## How it works
-
-BigCache relies on optimization presented in 1.5 version of Go ([issue-9477](https://github.com/golang/go/issues/9477)).
-This optimization states that if map without pointers in keys and values is used then GC will omit its content.
-Therefore BigCache uses `map[uint64]uint32` where keys are hashed and values are offsets of entries.
-
-Entries are kept in bytes array, to omit GC again.
-Bytes array size can grow to gigabytes without impact on performance
-because GC will only see single pointer to it.
-
-## Bigcache vs Freecache
-
-Both caches provide the same core features but they reduce GC overhead in different ways.
-Bigcache relies on `map[uint64]uint32`, freecache implements its own mapping built on
-slices to reduce number of pointers.
-
-Results from benchmark tests are presented above.
-One of the advantage of bigcache over freecache is that you don’t need to know
-the size of the cache in advance, because when bigcache is full,
-it can allocate additional memory for new entries instead of
-overwriting existing ones as freecache does currently.
-However hard max size in bigcache also can be set, check [HardMaxCacheSize](https://godoc.org/github.com/allegro/bigcache#Config).
-
-## HTTP Server
-
-This package also includes an easily deployable HTTP implementation of BigCache, which can be found in the [server](/server) package.
-
-## More
-
-Bigcache genesis is described in allegro.tech blog post: [writing a very fast cache service in Go](http://allegro.tech/2016/03/writing-fast-cache-service-in-go.html)
-
-## License
-
-BigCache is released under the Apache 2.0 license (see [LICENSE](LICENSE))

vendor/github.com/allegro/bigcache/bigcache.go

@@ -1,155 +0,0 @@
-package bigcache
-
-import (
-	"fmt"
-	"time"
-)
-
-const (
-	minimumEntriesInShard = 10 // Minimum number of entries in single shard
-)
-
-// BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance.
-// It keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place,
-// therefore entries (de)serialization in front of the cache will be needed in most use cases.
-type BigCache struct {
-	shards       []*cacheShard
-	lifeWindow   uint64
-	clock        clock
-	hash         Hasher
-	config       Config
-	shardMask    uint64
-	maxShardSize uint32
-}
-
-// NewBigCache initialize new instance of BigCache
-func NewBigCache(config Config) (*BigCache, error) {
-	return newBigCache(config, &systemClock{})
-}
-
-func newBigCache(config Config, clock clock) (*BigCache, error) {
-
-	if !isPowerOfTwo(config.Shards) {
-		return nil, fmt.Errorf("Shards number must be power of two")
-	}
-
-	if config.Hasher == nil {
-		config.Hasher = newDefaultHasher()
-	}
-
-	cache := &BigCache{
-		shards:       make([]*cacheShard, config.Shards),
-		lifeWindow:   uint64(config.LifeWindow.Seconds()),
-		clock:        clock,
-		hash:         config.Hasher,
-		config:       config,
-		shardMask:    uint64(config.Shards - 1),
-		maxShardSize: uint32(config.maximumShardSize()),
-	}
-
-	var onRemove func(wrappedEntry []byte)
-	if config.OnRemove == nil {
-		onRemove = cache.notProvidedOnRemove
-	} else {
-		onRemove = cache.providedOnRemove
-	}
-
-	for i := 0; i < config.Shards; i++ {
-		cache.shards[i] = initNewShard(config, onRemove, clock)
-	}
-
-	if config.CleanWindow > 0 {
-		go func() {
-			for t := range time.Tick(config.CleanWindow) {
-				cache.cleanUp(uint64(t.Unix()))
-			}
-		}()
-	}
-
-	return cache, nil
-}
-
-// Get reads entry for the key.
-// It returns an EntryNotFoundError when
-// no entry exists for the given key.
-func (c *BigCache) Get(key string) ([]byte, error) {
-	hashedKey := c.hash.Sum64(key)
-	shard := c.getShard(hashedKey)
-	return shard.get(key, hashedKey)
-}
-
-// Set saves entry under the key
-func (c *BigCache) Set(key string, entry []byte) error {
-	hashedKey := c.hash.Sum64(key)
-	shard := c.getShard(hashedKey)
-	return shard.set(key, hashedKey, entry)
-}
-
-// Delete removes the key
-func (c *BigCache) Delete(key string) error {
-	hashedKey := c.hash.Sum64(key)
-	shard := c.getShard(hashedKey)
-	return shard.del(key, hashedKey)
-}
-
-// Reset empties all cache shards
-func (c *BigCache) Reset() error {
-	for _, shard := range c.shards {
-		shard.reset(c.config)
-	}
-	return nil
-}
-
-// Len computes number of entries in cache
-func (c *BigCache) Len() int {
-	var len int
-	for _, shard := range c.shards {
-		len += shard.len()
-	}
-	return len
-}
-
-// Stats returns cache's statistics
-func (c *BigCache) Stats() Stats {
-	var s Stats
-	for _, shard := range c.shards {
-		tmp := shard.getStats()
-		s.Hits += tmp.Hits
-		s.Misses += tmp.Misses
-		s.DelHits += tmp.DelHits
-		s.DelMisses += tmp.DelMisses
-		s.Collisions += tmp.Collisions
-	}
-	return s
-}
-
-// Iterator returns iterator function to iterate over EntryInfo's from whole cache.
-func (c *BigCache) Iterator() *EntryInfoIterator {
-	return newIterator(c)
-}
-
-func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func() error) bool {
-	oldestTimestamp := readTimestampFromEntry(oldestEntry)
-	if currentTimestamp-oldestTimestamp > c.lifeWindow {
-		evict()
-		return true
-	}
-	return false
-}
-
-func (c *BigCache) cleanUp(currentTimestamp uint64) {
-	for _, shard := range c.shards {
-		shard.cleanUp(currentTimestamp)
-	}
-}
-
-func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) {
-	return c.shards[hashedKey&c.shardMask]
-}
-
-func (c *BigCache) providedOnRemove(wrappedEntry []byte) {
-	c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry))
-}
-
-func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte) {
-}

vendor/github.com/allegro/bigcache/bigcache_bench_test.go

@@ -1,141 +0,0 @@
-package bigcache
-
-import (
-	"fmt"
-	"math/rand"
-	"strconv"
-	"testing"
-	"time"
-)
-
-var message = blob('a', 256)
-
-func BenchmarkWriteToCacheWith1Shard(b *testing.B) {
-	writeToCache(b, 1, 100*time.Second, b.N)
-}
-
-func BenchmarkWriteToLimitedCacheWithSmallInitSizeAnd1Shard(b *testing.B) {
-	m := blob('a', 1024)
-	cache, _ := NewBigCache(Config{
-		Shards:             1,
-		LifeWindow:         100 * time.Second,
-		MaxEntriesInWindow: 100,
-		MaxEntrySize:       256,
-		HardMaxCacheSize:   1,
-	})
-
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		cache.Set(fmt.Sprintf("key-%d", i), m)
-	}
-}
-
-func BenchmarkWriteToUnlimitedCacheWithSmallInitSizeAnd1Shard(b *testing.B) {
-	m := blob('a', 1024)
-	cache, _ := NewBigCache(Config{
-		Shards:             1,
-		LifeWindow:         100 * time.Second,
-		MaxEntriesInWindow: 100,
-		MaxEntrySize:       256,
-	})
-
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		cache.Set(fmt.Sprintf("key-%d", i), m)
-	}
-}
-
-func BenchmarkWriteToCache(b *testing.B) {
-	for _, shards := range []int{1, 512, 1024, 8192} {
-		b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
-			writeToCache(b, shards, 100*time.Second, b.N)
-		})
-	}
-}
-
-func BenchmarkReadFromCache(b *testing.B) {
-	for _, shards := range []int{1, 512, 1024, 8192} {
-		b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
-			readFromCache(b, 1024)
-		})
-	}
-}
-
-func BenchmarkIterateOverCache(b *testing.B) {
-
-	m := blob('a', 1)
-
-	for _, shards := range []int{512, 1024, 8192} {
-		b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
-			cache, _ := NewBigCache(Config{
-				Shards:             shards,
-				LifeWindow:         1000 * time.Second,
-				MaxEntriesInWindow: max(b.N, 100),
-				MaxEntrySize:       500,
-			})
-
-			for i := 0; i < b.N; i++ {
-				cache.Set(fmt.Sprintf("key-%d", i), m)
-			}
-
-			b.ResetTimer()
-			it := cache.Iterator()
-
-			b.RunParallel(func(pb *testing.PB) {
-				b.ReportAllocs()
-
-				for pb.Next() {
-					if it.SetNext() {
-						it.Value()
-					}
-				}
-			})
-		})
-	}
-}
-
-func BenchmarkWriteToCacheWith1024ShardsAndSmallShardInitSize(b *testing.B) {
-	writeToCache(b, 1024, 100*time.Second, 100)
-}
-
-func writeToCache(b *testing.B, shards int, lifeWindow time.Duration, requestsInLifeWindow int) {
-	cache, _ := NewBigCache(Config{
-		Shards:             shards,
-		LifeWindow:         lifeWindow,
-		MaxEntriesInWindow: max(requestsInLifeWindow, 100),
-		MaxEntrySize:       500,
-	})
-	rand.Seed(time.Now().Unix())
-
-	b.RunParallel(func(pb *testing.PB) {
-		id := rand.Int()
-		counter := 0
-
-		b.ReportAllocs()
-		for pb.Next() {
-			cache.Set(fmt.Sprintf("key-%d-%d", id, counter), message)
-			counter = counter + 1
-		}
-	})
-}
-
-func readFromCache(b *testing.B, shards int) {
-	cache, _ := NewBigCache(Config{
-		Shards:             shards,
-		LifeWindow:         1000 * time.Second,
-		MaxEntriesInWindow: max(b.N, 100),
-		MaxEntrySize:       500,
-	})
-	for i := 0; i < b.N; i++ {
-		cache.Set(strconv.Itoa(i), message)
-	}
-	b.ResetTimer()
-
-	b.RunParallel(func(pb *testing.PB) {
-		b.ReportAllocs()
-
-		for pb.Next() {
-			cache.Get(strconv.Itoa(rand.Intn(b.N)))
-		}
-	})
-}

vendor/github.com/allegro/bigcache/bigcache_test.go

@@ -1,579 +0,0 @@
-package bigcache
-
-import (
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-var sink []byte
-
-func TestParallel(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(DefaultConfig(5 * time.Second))
-	value := []byte("value")
-	var wg sync.WaitGroup
-	wg.Add(3)
-	keys := 1337
-
-	// when
-	go func() {
-		defer wg.Done()
-		for i := 0; i < keys; i++ {
-			cache.Set(fmt.Sprintf("key%d", i), value)
-		}
-	}()
-	go func() {
-		defer wg.Done()
-		for i := 0; i < keys; i++ {
-			sink, _ = cache.Get(fmt.Sprintf("key%d", i))
-		}
-	}()
-	go func() {
-		defer wg.Done()
-		for i := 0; i < keys; i++ {
-			cache.Delete(fmt.Sprintf("key%d", i))
-		}
-	}()
-
-	// then
-	wg.Wait()
-}
-
-func TestWriteAndGetOnCache(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(DefaultConfig(5 * time.Second))
-	value := []byte("value")
-
-	// when
-	cache.Set("key", value)
-	cachedValue, err := cache.Get("key")
-
-	// then
-	assert.NoError(t, err)
-	assert.Equal(t, value, cachedValue)
-}
-
-func TestConstructCacheWithDefaultHasher(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             16,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 10,
-		MaxEntrySize:       256,
-	})
-
-	assert.IsType(t, fnv64a{}, cache.hash)
-}
-
-func TestWillReturnErrorOnInvalidNumberOfPartitions(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, error := NewBigCache(Config{
-		Shards:             18,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 10,
-		MaxEntrySize:       256,
-	})
-
-	assert.Nil(t, cache)
-	assert.Error(t, error, "Shards number must be power of two")
-}
-
-func TestEntryNotFound(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             16,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 10,
-		MaxEntrySize:       256,
-	})
-
-	// when
-	_, err := cache.Get("nonExistingKey")
-
-	// then
-	assert.EqualError(t, err, "Entry \"nonExistingKey\" not found")
-}
-
-func TestTimingEviction(t *testing.T) {
-	t.Parallel()
-
-	// given
-	clock := mockedClock{value: 0}
-	cache, _ := newBigCache(Config{
-		Shards:             1,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	}, &clock)
-
-	// when
-	cache.Set("key", []byte("value"))
-	clock.set(5)
-	cache.Set("key2", []byte("value2"))
-	_, err := cache.Get("key")
-
-	// then
-	assert.EqualError(t, err, "Entry \"key\" not found")
-}
-
-func TestTimingEvictionShouldEvictOnlyFromUpdatedShard(t *testing.T) {
-	t.Parallel()
-
-	// given
-	clock := mockedClock{value: 0}
-	cache, _ := newBigCache(Config{
-		Shards:             4,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	}, &clock)
-
-	// when
-	cache.Set("key", []byte("value"))
-	clock.set(5)
-	cache.Set("key2", []byte("value 2"))
-	value, err := cache.Get("key")
-
-	// then
-	assert.NoError(t, err, "Entry \"key\" not found")
-	assert.Equal(t, []byte("value"), value)
-}
-
-func TestCleanShouldEvictAll(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             4,
-		LifeWindow:         time.Second,
-		CleanWindow:        time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	})
-
-	// when
-	cache.Set("key", []byte("value"))
-	<-time.After(3 * time.Second)
-	value, err := cache.Get("key")
-
-	// then
-	assert.EqualError(t, err, "Entry \"key\" not found")
-	assert.Equal(t, value, []byte(nil))
-}
-
-func TestOnRemoveCallback(t *testing.T) {
-	t.Parallel()
-
-	// given
-	clock := mockedClock{value: 0}
-	onRemoveInvoked := false
-	onRemove := func(key string, entry []byte) {
-		onRemoveInvoked = true
-		assert.Equal(t, "key", key)
-		assert.Equal(t, []byte("value"), entry)
-	}
-	cache, _ := newBigCache(Config{
-		Shards:             1,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-		OnRemove:           onRemove,
-	}, &clock)
-
-	// when
-	cache.Set("key", []byte("value"))
-	clock.set(5)
-	cache.Set("key2", []byte("value2"))
-
-	// then
-	assert.True(t, onRemoveInvoked)
-}
-
-func TestCacheLen(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             8,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	})
-	keys := 1337
-
-	// when
-	for i := 0; i < keys; i++ {
-		cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
-	}
-
-	// then
-	assert.Equal(t, keys, cache.Len())
-}
-
-func TestCacheStats(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             8,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	})
-
-	// when
-	for i := 0; i < 100; i++ {
-		cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
-	}
-
-	for i := 0; i < 10; i++ {
-		value, err := cache.Get(fmt.Sprintf("key%d", i))
-		assert.Nil(t, err)
-		assert.Equal(t, string(value), "value")
-	}
-	for i := 100; i < 110; i++ {
-		_, err := cache.Get(fmt.Sprintf("key%d", i))
-		assert.Error(t, err)
-	}
-	for i := 10; i < 20; i++ {
-		err := cache.Delete(fmt.Sprintf("key%d", i))
-		assert.Nil(t, err)
-	}
-	for i := 110; i < 120; i++ {
-		err := cache.Delete(fmt.Sprintf("key%d", i))
-		assert.Error(t, err)
-	}
-
-	// then
-	stats := cache.Stats()
-	assert.Equal(t, stats.Hits, int64(10))
-	assert.Equal(t, stats.Misses, int64(10))
-	assert.Equal(t, stats.DelHits, int64(10))
-	assert.Equal(t, stats.DelMisses, int64(10))
-}
-
-func TestCacheDel(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(DefaultConfig(time.Second))
-
-	// when
-	err := cache.Delete("nonExistingKey")
-
-	// then
-	assert.Equal(t, err.Error(), "Entry \"nonExistingKey\" not found")
-
-	// and when
-	cache.Set("existingKey", nil)
-	err = cache.Delete("existingKey")
-	cachedValue, _ := cache.Get("existingKey")
-
-	// then
-	assert.Nil(t, err)
-	assert.Len(t, cachedValue, 0)
-}
-
-func TestCacheReset(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             8,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	})
-	keys := 1337
-
-	// when
-	for i := 0; i < keys; i++ {
-		cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
-	}
-
-	// then
-	assert.Equal(t, keys, cache.Len())
-
-	// and when
-	cache.Reset()
-
-	// then
-	assert.Equal(t, 0, cache.Len())
-
-	// and when
-	for i := 0; i < keys; i++ {
-		cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
-	}
-
-	// then
-	assert.Equal(t, keys, cache.Len())
-}
-
-func TestIterateOnResetCache(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             8,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	})
-	keys := 1337
-
-	// when
-	for i := 0; i < keys; i++ {
-		cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
-	}
-	cache.Reset()
-
-	// then
-	iterator := cache.Iterator()
-
-	assert.Equal(t, false, iterator.SetNext())
-}
-
-func TestGetOnResetCache(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             8,
-		LifeWindow:         time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	})
-	keys := 1337
-
-	// when
-	for i := 0; i < keys; i++ {
-		cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
-	}
-
-	cache.Reset()
-
-	// then
-	value, err := cache.Get("key1")
-
-	assert.Equal(t, err.Error(), "Entry \"key1\" not found")
-	assert.Equal(t, value, []byte(nil))
-}
-
-func TestEntryUpdate(t *testing.T) {
-	t.Parallel()
-
-	// given
-	clock := mockedClock{value: 0}
-	cache, _ := newBigCache(Config{
-		Shards:             1,
-		LifeWindow:         6 * time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       256,
-	}, &clock)
-
-	// when
-	cache.Set("key", []byte("value"))
-	clock.set(5)
-	cache.Set("key", []byte("value2"))
-	clock.set(7)
-	cache.Set("key2", []byte("value3"))
-	cachedValue, _ := cache.Get("key")
-
-	// then
-	assert.Equal(t, []byte("value2"), cachedValue)
-}
-
-func TestOldestEntryDeletionWhenMaxCacheSizeIsReached(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             1,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       1,
-		HardMaxCacheSize:   1,
-	})
-
-	// when
-	cache.Set("key1", blob('a', 1024*400))
-	cache.Set("key2", blob('b', 1024*400))
-	cache.Set("key3", blob('c', 1024*800))
-
-	_, key1Err := cache.Get("key1")
-	_, key2Err := cache.Get("key2")
-	entry3, _ := cache.Get("key3")
-
-	// then
-	assert.EqualError(t, key1Err, "Entry \"key1\" not found")
-	assert.EqualError(t, key2Err, "Entry \"key2\" not found")
-	assert.Equal(t, blob('c', 1024*800), entry3)
-}
-
-func TestRetrievingEntryShouldCopy(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             1,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       1,
-		HardMaxCacheSize:   1,
-	})
-	cache.Set("key1", blob('a', 1024*400))
-	value, key1Err := cache.Get("key1")
-
-	// when
-	// override queue
-	cache.Set("key2", blob('b', 1024*400))
-	cache.Set("key3", blob('c', 1024*400))
-	cache.Set("key4", blob('d', 1024*400))
-	cache.Set("key5", blob('d', 1024*400))
-
-	// then
-	assert.Nil(t, key1Err)
-	assert.Equal(t, blob('a', 1024*400), value)
-}
-
-func TestEntryBiggerThanMaxShardSizeError(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             1,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       1,
-		HardMaxCacheSize:   1,
-	})
-
-	// when
-	err := cache.Set("key1", blob('a', 1024*1025))
-
-	// then
-	assert.EqualError(t, err, "entry is bigger than max shard size")
-}
-
-func TestHashCollision(t *testing.T) {
-	t.Parallel()
-
-	ml := &mockedLogger{}
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             16,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 10,
-		MaxEntrySize:       256,
-		Verbose:            true,
-		Hasher:             hashStub(5),
-		Logger:             ml,
-	})
-
-	// when
-	cache.Set("liquid", []byte("value"))
-	cachedValue, err := cache.Get("liquid")
-
-	// then
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("value"), cachedValue)
-
-	// when
-	cache.Set("costarring", []byte("value 2"))
-	cachedValue, err = cache.Get("costarring")
-
-	// then
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("value 2"), cachedValue)
-
-	// when
-	cachedValue, err = cache.Get("liquid")
-
-	// then
-	assert.Error(t, err)
-	assert.Nil(t, cachedValue)
-
-	assert.NotEqual(t, "", ml.lastFormat)
-	assert.Equal(t, cache.Stats().Collisions, int64(1))
-}
-
-func TestNilValueCaching(t *testing.T) {
-	t.Parallel()
-
-	// given
-	cache, _ := NewBigCache(Config{
-		Shards:             1,
-		LifeWindow:         5 * time.Second,
-		MaxEntriesInWindow: 1,
-		MaxEntrySize:       1,
-		HardMaxCacheSize:   1,
-	})
-
-	// when
-	cache.Set("Kierkegaard", []byte{})
-	cachedValue, err := cache.Get("Kierkegaard")
-
-	// then
-	assert.NoError(t, err)
-	assert.Equal(t, []byte{}, cachedValue)
-
-	// when
-	cache.Set("Sartre", nil)
-	cachedValue, err = cache.Get("Sartre")
-
-	// then
-	assert.NoError(t, err)
-	assert.Equal(t, []byte{}, cachedValue)
-
-	// when
-	cache.Set("Nietzsche", []byte(nil))
-	cachedValue, err = cache.Get("Nietzsche")
-
-	// then
-	assert.NoError(t, err)
-	assert.Equal(t, []byte{}, cachedValue)
-}
-
-type mockedLogger struct {
-	lastFormat string
-	lastArgs   []interface{}
-}
-
-func (ml *mockedLogger) Printf(format string, v ...interface{}) {
-	ml.lastFormat = format
-	ml.lastArgs = v
-}
-
-type mockedClock struct {
-	value int64
-}
-
-func (mc *mockedClock) epoch() int64 {
-	return mc.value
-}
-
-func (mc *mockedClock) set(value int64) {
-	mc.value = value
-}
-
-func blob(char byte, len int) []byte {
-	b := make([]byte, len)
-	for index := range b {
-		b[index] = char
-	}
-	return b
-}

vendor/github.com/allegro/bigcache/caches_bench/caches_bench_test.go

@@ -1,219 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"math/rand"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/allegro/bigcache"
-	"github.com/coocood/freecache"
-)
-
-const maxEntrySize = 256
-
-func BenchmarkMapSet(b *testing.B) {
-	m := make(map[string][]byte)
-	for i := 0; i < b.N; i++ {
-		m[key(i)] = value()
-	}
-}
-
-func BenchmarkConcurrentMapSet(b *testing.B) {
-	var m sync.Map
-	for i := 0; i < b.N; i++ {
-		m.Store(key(i), value())
-	}
-}
-
-func BenchmarkFreeCacheSet(b *testing.B) {
-	cache := freecache.NewCache(b.N * maxEntrySize)
-	for i := 0; i < b.N; i++ {
-		cache.Set([]byte(key(i)), value(), 0)
-	}
-}
-
-func BenchmarkBigCacheSet(b *testing.B) {
-	cache := initBigCache(b.N)
-	for i := 0; i < b.N; i++ {
-		cache.Set(key(i), value())
-	}
-}
-
-func BenchmarkMapGet(b *testing.B) {
-	b.StopTimer()
-	m := make(map[string][]byte)
-	for i := 0; i < b.N; i++ {
-		m[key(i)] = value()
-	}
-
-	b.StartTimer()
-	hitCount := 0
-	for i := 0; i < b.N; i++ {
-		if m[key(i)] != nil {
-			hitCount++
-		}
-	}
-}
-
-func BenchmarkConcurrentMapGet(b *testing.B) {
-	b.StopTimer()
-	var m sync.Map
-	for i := 0; i < b.N; i++ {
-		m.Store(key(i), value())
-	}
-
-	b.StartTimer()
-	hitCounter := 0
-	for i := 0; i < b.N; i++ {
-		_, ok := m.Load(key(i))
-		if ok {
-			hitCounter++
-		}
-	}
-}
-
-func BenchmarkFreeCacheGet(b *testing.B) {
-	b.StopTimer()
-	cache := freecache.NewCache(b.N * maxEntrySize)
-	for i := 0; i < b.N; i++ {
-		cache.Set([]byte(key(i)), value(), 0)
-	}
-
-	b.StartTimer()
-	for i := 0; i < b.N; i++ {
-		cache.Get([]byte(key(i)))
-	}
-}
-
-func BenchmarkBigCacheGet(b *testing.B) {
-	b.StopTimer()
-	cache := initBigCache(b.N)
-	for i := 0; i < b.N; i++ {
-		cache.Set(key(i), value())
-	}
-
-	b.StartTimer()
-	for i := 0; i < b.N; i++ {
-		cache.Get(key(i))
-	}
-}
-
-func BenchmarkBigCacheSetParallel(b *testing.B) {
-	cache := initBigCache(b.N)
-	rand.Seed(time.Now().Unix())
-
-	b.RunParallel(func(pb *testing.PB) {
-		id := rand.Intn(1000)
-		counter := 0
-		for pb.Next() {
-			cache.Set(parallelKey(id, counter), value())
-			counter = counter + 1
-		}
-	})
-}
-
-func BenchmarkFreeCacheSetParallel(b *testing.B) {
-	cache := freecache.NewCache(b.N * maxEntrySize)
-	rand.Seed(time.Now().Unix())
-
-	b.RunParallel(func(pb *testing.PB) {
-		id := rand.Intn(1000)
-		counter := 0
-		for pb.Next() {
-			cache.Set([]byte(parallelKey(id, counter)), value(), 0)
-			counter = counter + 1
-		}
-	})
-}
-
-func BenchmarkConcurrentMapSetParallel(b *testing.B) {
-	var m sync.Map
-
-	b.RunParallel(func(pb *testing.PB) {
-		id := rand.Intn(1000)
-		for pb.Next() {
-			m.Store(key(id), value())
-		}
-	})
-}
-
-func BenchmarkBigCacheGetParallel(b *testing.B) {
-	b.StopTimer()
-	cache := initBigCache(b.N)
-	for i := 0; i < b.N; i++ {
-		cache.Set(key(i), value())
-	}
-
-	b.StartTimer()
-	b.RunParallel(func(pb *testing.PB) {
-		counter := 0
-		for pb.Next() {
-			cache.Get(key(counter))
-			counter = counter + 1
-		}
-	})
-}
-
-func BenchmarkFreeCacheGetParallel(b *testing.B) {
-	b.StopTimer()
-	cache := freecache.NewCache(b.N * maxEntrySize)
-	for i := 0; i < b.N; i++ {
-		cache.Set([]byte(key(i)), value(), 0)
-	}
-
-	b.StartTimer()
-	b.RunParallel(func(pb *testing.PB) {
-		counter := 0
-		for pb.Next() {
-			cache.Get([]byte(key(counter)))
-			counter = counter + 1
-		}
-	})
-}
-
-func BenchmarkConcurrentMapGetParallel(b *testing.B) {
-	b.StopTimer()
-	var m sync.Map
-	for i := 0; i < b.N; i++ {
-		m.Store(key(i), value())
-	}
-
-	b.StartTimer()
-	hitCount := 0
-
-	b.RunParallel(func(pb *testing.PB) {
-		id := rand.Intn(1000)
-		for pb.Next() {
-			_, ok := m.Load(key(id))
-			if ok {
-				hitCount++
-			}
-		}
-	})
-}
-
-func key(i int) string {
-	return fmt.Sprintf("key-%010d", i)
-}
-
-func value() []byte {
-	return make([]byte, 100)
-}
-
-func parallelKey(threadID int, counter int) string {
-	return fmt.Sprintf("key-%04d-%06d", threadID, counter)
-}
-
-func initBigCache(entriesInWindow int) *bigcache.BigCache {
-	cache, _ := bigcache.NewBigCache(bigcache.Config{
-		Shards:             256,
-		LifeWindow:         10 * time.Minute,
-		MaxEntriesInWindow: entriesInWindow,
-		MaxEntrySize:       maxEntrySize,
-		Verbose:            true,
-	})
-
-	return cache
-}