Compare commits

...
90 Commits
Author SHA1 Message Date
Elizabeth b6a2a278ce Merge pull request #126 from vulcanize/vdb-585-create-address-table
Vdb-585 create address table
2019-08-26 15:13:07 -05:00
Elizabeth Engelman edc0bdf668 Address more PR feedback 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 1b3786338f Address PR feedback 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 58c23c6632 Update Copyrights 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 38f6847ff2 Create a HeaderSyncReceiptRepository 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 7e38764618 Rename ReceiptRepository -> FullSyncReceiptRepository 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 4e40e892d2 Updating header_sync_receipts to have FK reference to addresses 2019-08-23 10:11:37 -05:00
Elizabeth Engelman bcd6d14fcd Updating full_sync_receipts to have FK reference to addresses 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 258035833b Add GetAddressById 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 4b61c87b55 Get or create address record in a transaction 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 1373fe83a1 Pass a db into GetOrCreateAddress 2019-08-23 10:11:37 -05:00
Elizabeth Engelman b13eefbce7 Add AddressRepository 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 3005a1ad50 Move add address migration table up in the migration order 2019-08-23 10:11:37 -05:00
Elizabeth Engelman 750754caa7 Add an addresses table 2019-08-23 10:11:37 -05:00
Rob Mulholand e093ad351f Merge pull request #130 from vulcanize/update-deps
Update dependencies
2019-08-21 16:04:55 -05:00
Rob Mulholand 33e7651096 Update dependencies 2019-08-20 16:17:25 -05:00
Gabe Laughlin baea9740c1 Merge pull request #125 from vulcanize/decode-packed-addresses
(VDB-768) Allow addresses in packed slots to be decoded
2019-08-05 21:40:15 -05:00
Gabe Laughlin 4f8b057726 (VDB-768) Allow addresses in packed slots to be decoded 2019-08-01 22:40:50 -05:00
Andrew J Yao 65e762f80b Merge pull request #124 from vulcanize/refactor-mcd-configs
Removes _checked from column name query
2019-07-30 18:18:42 -07:00
Andrew J Yao dc8d52a58d Removes _checked from column name query 2019-07-30 18:07:25 -07:00
Rob Mulholand f2547b5a42 Merge pull request #117 from vulcanize/geth-1.9.0
Update Geth to 1.9.0
2019-07-29 15:04:37 -05:00
Rob Mulholand 62e454b7fe Update docs to reflect using go mod 2019-07-23 15:28:08 -05:00
Rob Mulholand ca4451d914 Update Travis to enable go mod
- set GO111MODULE=on
- use go 1.12
2019-07-23 15:28:08 -05:00
Rob Mulholand f51f5c64a0 Replace dep with go mod 2019-07-23 15:28:08 -05:00
Rob Mulholand 987abd4b2e Update Geth to 1.9.0 2019-07-23 15:26:18 -05:00
Andrew J Yao a3b55bfd56 Merge pull request #120 from vulcanize/logging-enhancement
Logging enhancement
2019-07-23 13:22:58 -07:00
Andrew J Yao ee77fc6521 Remove injection of SubCommand for logs 2019-07-23 13:11:20 -07:00
Elizabeth 85aba29cf2 Merge pull request #121 from vulcanize/allow-for-packed-storage-slots
VDB-318 Allow for packed storage slots
2019-07-23 13:48:36 -05:00
Elizabeth Engelman 2c66afbcd7 Address PR feedback
- factor out bitsPerByte constant
- panic if the necessary args aren't passed to GetStorageValueMetadataForPackedSlot
- only have one decode integer method in the deocoder file
2019-07-23 13:36:58 -05:00
Andrew J Yao a188e1dd79 Incorporate changes from Edvard PR 2019-07-23 08:10:58 -07:00
Andrew J Yao 92d153b010 Add subCommand name to log fields 2019-07-23 08:09:57 -07:00
Andrew J Yao d7a82e353b Add calling method to debug level log messages 2019-07-23 08:08:15 -07:00
Elizabeth Engelman 69ad521482 Apply go fmt changes and cleanup 2019-07-19 10:44:36 -05:00
Elizabeth Engelman d30b4ea4f2 Update GetStorageValueMetadata 2019-07-19 10:13:30 -05:00
Elizabeth Engelman 51915f7506 Pass storage values to repo.create as a map 2019-07-18 18:40:38 -05:00
Elizabeth Engelman ebfb4965c7 Add tests for decoding packed storage 2019-07-18 16:21:57 -05:00
Elizabeth Engelman b0fff9a1dd First pass at decoded packed storage slots 2019-07-18 13:30:04 -05:00
Edvard Hübinette 2c092e8d04 [VDB-751 VDB-754] Bugfix null pointer panic and improve logging (#119)
* VDB-751 VDB-754 Bugfix null pointer panic and improve logging

* Fix typo
2019-07-18 09:21:40 +02:00
Rob Mulholand af6190ea09 Merge pull request #113 from vulcanize/plugin-updates
Plugin updates
2019-06-27 14:11:12 -05:00
Rob Mulholand 0f95267e84 Use goose library for running plugin migrations
- Removes dependency on goose binary existing in plugin execution
  environment
2019-06-27 13:58:58 -05:00
Rob Mulholand f0bcf930f4 Separate tmp migrations dir from home in join
- enable separator between home and migrations dir
2019-06-27 13:58:58 -05:00
Rob Mulholand 0558792282 Log error on failure to setup plugin migrations
- include specific error
2019-06-27 13:58:58 -05:00
Rob Mulholand 62cca62432 Log error on failure to read config file
- include actual error to highlight the specific issue
2019-06-27 13:58:58 -05:00
Andrew J Yao 7092d33338 Merge pull request #112 from vulcanize/vdb-605-add-debug-logging
Adds ability to set log level from toml or CLI args
2019-06-27 11:56:47 -07:00
Andrew J Yao 11a65c525c Adds ability to set log level from toml or CLI args 2019-06-27 11:14:11 -07:00
Elizabeth 3d57f6030d Merge pull request #106 from vulcanize/update-docs
Remove repository-maintenance.md as this info now lives in contributi…
2019-06-26 13:45:54 -05:00
Elizabeth Engelman b124bea974 Remove repository-maintenance.md as this info now lives in contributing.md 2019-06-18 14:42:20 -05:00
Rob Mulholand f0a7a7d999 Merge pull request #103 from vulcanize/remove-queued-storage-duplicates
Don't duplicate queued storage diffs
2019-06-17 15:55:35 -05:00
Rob Mulholand 7d6629e5b4 Merge pull request #102 from vulcanize/fix-mappings-util
Fix mappings util
2019-06-17 15:45:55 -05:00
Elizabeth d761f88b30 Merge pull request #101 from vulcanize/update-transformer-execute
VDB-696 Update transformer execute
2019-06-17 15:19:51 -05:00
Elizabeth Engelman 948246cf83 Address PR comments 2019-06-17 14:49:25 -05:00
Rob Mulholand e11f2c8c59 Don't duplicate queued storage diffs
- currently, if we don't recognize the same diff several times (e.g.
  if you restart the storage diff watcher pointed at the same file),
  we'll add the same row to the queue on each run.
- these changes assure we only queue an unrecognized diff once.
2019-06-14 11:26:15 -05:00
Rob Mulholand 95fa6280c5 Fix mappings util
- Remove hex prefix so that the function can accept keys with or
  without a hex prefix and return the same result
2019-06-13 18:33:22 -05:00
Elizabeth Engelman fd126c7bea Remove unused RecheckHeaders method and update Repo tests 2019-06-13 15:19:23 -05:00
Elizabeth Engelman f292888195 Refactor checked column specific SQL 2019-06-13 12:36:29 -05:00
Elizabeth Engelman adf91df23a Remove unused RecheckHeaders method from the events Repository interface 2019-06-13 10:33:53 -05:00
Elizabeth Engelman ede6ad1c8f Remove unused MissingHeaders method from the events Repository interface 2019-06-13 10:33:48 -05:00
Elizabeth Engelman ee13d5c1df Remove unused recheckHeaders bool argument from Transformer.Execute 2019-06-13 10:33:16 -05:00
Elizabeth eb2b3a0263 Merge pull request #98 from vulcanize/additional-doc-updates
Move changelog and diagram information to contributing.md
2019-06-03 14:24:43 -05:00
Elizabeth Engelman 5088574437 Address PR comment 2019-06-03 13:36:56 -05:00
Elizabeth Engelman d61f4f3cb4 Move changelog and diagram information to contributing.md 2019-06-03 13:31:50 -05:00
Elizabeth 1a455741ce Merge pull request #97 from vulcanize/quick-link-fix
Fixes broken link in docs
2019-06-03 13:30:51 -05:00
yaoandrew 180c2d963e Fixes broken link in docs 2019-05-28 15:24:04 -07:00
Rob Mulholand 2598704690 Merge pull request #96 from vulcanize/remove-storage-diff-duplicate-handling
Remove handling of duplicate storage diffs in watcher
2019-05-20 14:04:50 -05:00
Rob Mulholand e2909797fc Remove handling of duplicate storage diffs in watcher
- Can push this responsibility down to the transformers
- Update docs to reflect that transformers should handle duplicates
2019-05-20 13:29:09 -05:00
Elizabeth 79765c7998 Merge pull request #94 from vulcanize/documentation-updates
Documentation updates
2019-05-15 13:22:15 -05:00
Gabe Laughlin 9c6549258e Merge pull request #95 from vulcanize/vdb-479-upgrade-postgres
(VDB-479) Upgrade Posgres to 11
2019-05-15 12:29:06 -05:00
Gabe Laughlin b4cdf76905 (VDB-479) Upgrade Posgres to 11 2019-05-14 16:49:05 -05:00
Elizabeth Engelman 622ea44a6f Mention reorgs in data-sync documentation 2019-05-13 16:34:00 -05:00
Elizabeth Engelman d947c8f30a Rename sync command to fullSync 2019-05-13 16:14:56 -05:00
Elizabeth Engelman 481988cb08 Update date-syncing doc 2019-05-13 16:14:56 -05:00
Elizabeth Engelman 4580f0fc2e Updates to custom-transformers doc 2019-05-13 16:14:56 -05:00
Elizabeth Engelman eba52442ec Restructure README to conform with standard readme spec 2019-05-13 16:14:56 -05:00
Elizabeth Engelman 7ddf728db2 Descript different custom sync transformer types 2019-05-13 13:40:46 -05:00
Elizabeth Engelman 829a581727 Rename sync documentation file 2019-05-13 13:31:23 -05:00
Elizabeth Engelman fa03716cb2 Address small PR comments 2019-05-10 11:54:15 -05:00
Elizabeth Engelman bd3e841e21 Update links in README to be relative to current branch 2019-05-10 11:01:17 -05:00
Elizabeth Engelman 8dc9ddf65e Add Code of Conduct 2019-05-10 10:07:25 -05:00
Elizabeth Engelman 7d1b334bda Update postgraphile documentation to mention no-ignore-rbac flag 2019-05-10 09:50:28 -05:00
Elizabeth Engelman ade1429ce3 Update transformer documentation
broken into generic and custom
2019-05-08 17:29:24 -05:00
Elizabeth Engelman 5d1ba5903c Update contributing guidelines 2019-05-08 13:42:43 -05:00
Elizabeth Engelman a49f5d7582 Add repository maintenance documentation 2019-05-08 13:42:43 -05:00
Elizabeth Engelman 436d9b9cb1 Add VDB overview diagram to README 2019-05-08 13:42:43 -05:00
Elizabeth Engelman f7d520c933 Update header sync transformer alias in ContractWatcher 2019-05-08 13:42:43 -05:00
Elizabeth Engelman e1a0d894a2 Update blockchain method GetHeaderByNumbers -> GetHeadersByNumbers 2019-05-08 13:42:43 -05:00
Elizabeth Engelman ba81766f6c Small spelling fix 2019-05-08 13:42:43 -05:00
Elizabeth Engelman b96f6c4d31 Various README updates 2019-05-08 13:42:43 -05:00
Rob Mulholand c5cfa85f4c Merge pull request #90 from vulcanize/vdb-563-remove-postgraphile
(VDB-563) Remove Postgraphile bespoke code
2019-05-08 12:51:56 -05:00
Rob Mulholand 702e839f5b Update Travis to reflect removing Postgraphile 2019-05-08 09:11:51 -05:00
Rob Mulholand ca354de229 (VDB-563) Remove Postgraphile bespoke code
- Add documentation for running Postgraphile from their binary
2019-05-08 09:11:51 -05:00
5587 changed files with 2047 additions and 1842807 deletions
+5 -8
View File
@@ -1,27 +1,24 @@
dist: trusty
language: go
go:
- 1.11
- 1.12
services:
- postgresql
addons:
postgresql: '9.6'
postgresql: '11.2'
go_import_path: github.com/vulcanize/vulcanizedb
before_install:
- make installtools
- bash ./scripts/install-postgres-10.sh
- bash ./scripts/install-postgres-11.sh
- curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
- echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
- sudo apt-get update && sudo apt-get install yarn
before_script:
- sudo -u postgres createdb vulcanize_private
- make migrate NAME=vulcanize_private
- cd postgraphile && yarn
script:
- yarn test
- cd ../
- make test
- make integrationtest
- env GO111MODULE=on make test
- env GO111MODULE=on make integrationtest
notifications:
email: false
env:
Generated
-563
View File
@@ -1,563 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:48a213e9dc4880bbbd6999309a476fa4d3cc67560aa7127154cf8ea95bd464c2"
name = "github.com/allegro/bigcache"
packages = [
".",
"queue",
]
pruneopts = ""
revision = "f31987a23e44c5121ef8c8b2f2ea2e8ffa37b068"
version = "v1.1.0"
[[projects]]
branch = "master"
digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
name = "github.com/aristanetworks/goarista"
packages = ["monotime"]
pruneopts = ""
revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
[[projects]]
branch = "master"
digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
name = "github.com/btcsuite/btcd"
packages = ["btcec"]
pruneopts = ""
revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
[[projects]]
digest = "1:5d47691333460db6ac83ced03c79b4bdb9aff3e322be24affb7855bed8affc6c"
name = "github.com/dave/jennifer"
packages = ["jen"]
pruneopts = ""
revision = "14e399b6b5e8456c66c45c955fc27b568bacb5c9"
version = "v1.3.0"
[[projects]]
digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
name = "github.com/deckarep/golang-set"
packages = ["."]
pruneopts = ""
revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
version = "v1.7.1"
[[projects]]
digest = "1:90d36f5b581e95e00ced808cd48824ed6c320c25887828cce461bdef4cb7bc7c"
name = "github.com/ethereum/go-ethereum"
packages = [
".",
"accounts",
"accounts/abi",
"accounts/abi/bind",
"accounts/keystore",
"common",
"common/bitutil",
"common/hexutil",
"common/math",
"common/mclock",
"common/prque",
"core/rawdb",
"core/types",
"crypto",
"crypto/ecies",
"crypto/secp256k1",
"ethclient",
"ethdb",
"ethdb/leveldb",
"ethdb/memorydb",
"event",
"log",
"metrics",
"p2p",
"p2p/discover",
"p2p/discv5",
"p2p/enode",
"p2p/enr",
"p2p/nat",
"p2p/netutil",
"params",
"rlp",
"rpc",
"trie",
]
pruneopts = ""
revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
[[projects]]
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
name = "github.com/fsnotify/fsnotify"
packages = ["."]
pruneopts = ""
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
version = "v1.4.7"
[[projects]]
digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
name = "github.com/go-stack/stack"
packages = ["."]
pruneopts = ""
revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
version = "v1.8.0"
[[projects]]
digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
name = "github.com/golang/protobuf"
packages = ["proto"]
pruneopts = ""
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
name = "github.com/golang/snappy"
packages = ["."]
pruneopts = ""
revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
[[projects]]
digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
name = "github.com/google/uuid"
packages = ["."]
pruneopts = ""
revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
name = "github.com/hashicorp/golang-lru"
packages = [
".",
"simplelru",
]
pruneopts = ""
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
[[projects]]
digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
name = "github.com/hashicorp/hcl"
packages = [
".",
"hcl/ast",
"hcl/parser",
"hcl/printer",
"hcl/scanner",
"hcl/strconv",
"hcl/token",
"json/parser",
"json/scanner",
"json/token",
]
pruneopts = ""
revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
version = "v1.0.0"
[[projects]]
digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
name = "github.com/hpcloud/tail"
packages = [
".",
"ratelimiter",
"util",
"watch",
"winfile",
]
pruneopts = ""
revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
name = "github.com/huin/goupnp"
packages = [
".",
"dcps/internetgateway1",
"dcps/internetgateway2",
"httpu",
"scpd",
"soap",
"ssdp",
]
pruneopts = ""
revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
[[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
pruneopts = ""
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
name = "github.com/jackpal/go-nat-pmp"
packages = ["."]
pruneopts = ""
revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
name = "github.com/jmoiron/sqlx"
packages = [
".",
"reflectx",
]
pruneopts = ""
revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
[[projects]]
digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3"
name = "github.com/konsorten/go-windows-terminal-sequences"
packages = ["."]
pruneopts = ""
revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242"
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
name = "github.com/lib/pq"
packages = [
".",
"oid",
]
pruneopts = ""
revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
[[projects]]
digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
name = "github.com/magiconair/properties"
packages = ["."]
pruneopts = ""
revision = "c2353362d570a7bfa228149c62842019201cfb71"
version = "v1.8.0"
[[projects]]
digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
name = "github.com/mitchellh/go-homedir"
packages = ["."]
pruneopts = ""
revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
version = "v1.0.0"
[[projects]]
digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
name = "github.com/mitchellh/mapstructure"
packages = ["."]
pruneopts = ""
revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
version = "v1.0.0"
[[projects]]
digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
name = "github.com/onsi/ginkgo"
packages = [
".",
"config",
"internal/codelocation",
"internal/containernode",
"internal/failer",
"internal/leafnodes",
"internal/remote",
"internal/spec",
"internal/spec_iterator",
"internal/specrunner",
"internal/suite",
"internal/testingtproxy",
"internal/writer",
"reporters",
"reporters/stenographer",
"reporters/stenographer/support/go-colorable",
"reporters/stenographer/support/go-isatty",
"types",
]
pruneopts = ""
revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
version = "v1.6.0"
[[projects]]
digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
name = "github.com/onsi/gomega"
packages = [
".",
"format",
"ghttp",
"internal/assertion",
"internal/asyncassertion",
"internal/oraclematcher",
"internal/testingtsupport",
"matchers",
"matchers/support/goraph/bipartitegraph",
"matchers/support/goraph/edge",
"matchers/support/goraph/node",
"matchers/support/goraph/util",
"types",
]
pruneopts = ""
revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
version = "v1.4.2"
[[projects]]
digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
name = "github.com/pborman/uuid"
packages = ["."]
pruneopts = ""
revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
version = "v1.2"
[[projects]]
digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
name = "github.com/pelletier/go-toml"
packages = ["."]
pruneopts = ""
revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
version = "v1.2.0"
[[projects]]
digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
name = "github.com/rjeczalik/notify"
packages = ["."]
pruneopts = ""
revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
version = "v0.9.1"
[[projects]]
digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
name = "github.com/rs/cors"
packages = ["."]
pruneopts = ""
revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
version = "v1.5.0"
[[projects]]
digest = "1:9d57e200ef5ccc4217fe0a34287308bac652435e7c6513f6263e0493d2245c56"
name = "github.com/sirupsen/logrus"
packages = ["."]
pruneopts = ""
revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95"
version = "v1.2.0"
[[projects]]
digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
name = "github.com/spf13/afero"
packages = [
".",
"mem",
]
pruneopts = ""
revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
version = "v1.1.2"
[[projects]]
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
name = "github.com/spf13/cast"
packages = ["."]
pruneopts = ""
revision = "8965335b8c7107321228e3e3702cab9832751bac"
version = "v1.2.0"
[[projects]]
digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
name = "github.com/spf13/cobra"
packages = ["."]
pruneopts = ""
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
version = "v0.0.3"
[[projects]]
digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
name = "github.com/spf13/jwalterweatherman"
packages = ["."]
pruneopts = ""
revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
version = "v1.0.0"
[[projects]]
digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
name = "github.com/spf13/pflag"
packages = ["."]
pruneopts = ""
revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
version = "v1.0.2"
[[projects]]
digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
name = "github.com/spf13/viper"
packages = ["."]
pruneopts = ""
revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
name = "github.com/syndtr/goleveldb"
packages = [
"leveldb",
"leveldb/cache",
"leveldb/comparer",
"leveldb/errors",
"leveldb/filter",
"leveldb/iterator",
"leveldb/journal",
"leveldb/memdb",
"leveldb/opt",
"leveldb/storage",
"leveldb/table",
"leveldb/util",
]
pruneopts = ""
revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
[[projects]]
branch = "master"
digest = "1:59b49c47c11a48f1054529207f65907c014ecf5f9a7c0d9c0f1616dec7b062ed"
name = "golang.org/x/crypto"
packages = [
"pbkdf2",
"scrypt",
"sha3",
"ssh/terminal",
]
pruneopts = ""
revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
[[projects]]
branch = "master"
digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
name = "golang.org/x/net"
packages = [
"context",
"html",
"html/atom",
"html/charset",
"websocket",
]
pruneopts = ""
revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
[[projects]]
branch = "master"
digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
name = "golang.org/x/sync"
packages = ["errgroup"]
pruneopts = ""
revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
[[projects]]
branch = "master"
digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
name = "golang.org/x/sys"
packages = [
"unix",
"windows",
]
pruneopts = ""
revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
[[projects]]
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
name = "golang.org/x/text"
packages = [
"encoding",
"encoding/charmap",
"encoding/htmlindex",
"encoding/internal",
"encoding/internal/identifier",
"encoding/japanese",
"encoding/korean",
"encoding/simplifiedchinese",
"encoding/traditionalchinese",
"encoding/unicode",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"internal/utf8internal",
"language",
"runes",
"transform",
"unicode/cldr",
"unicode/norm",
]
pruneopts = ""
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
name = "gopkg.in/fsnotify.v1"
packages = ["."]
pruneopts = ""
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
source = "gopkg.in/fsnotify/fsnotify.v1"
version = "v1.4.7"
[[projects]]
branch = "v2"
digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
name = "gopkg.in/natefinch/npipe.v2"
packages = ["."]
pruneopts = ""
revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
[[projects]]
branch = "v1"
digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
name = "gopkg.in/tomb.v1"
packages = ["."]
pruneopts = ""
revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
[[projects]]
digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = ""
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/dave/jennifer/jen",
"github.com/ethereum/go-ethereum",
"github.com/ethereum/go-ethereum/accounts/abi",
"github.com/ethereum/go-ethereum/accounts/abi/bind",
"github.com/ethereum/go-ethereum/common",
"github.com/ethereum/go-ethereum/common/hexutil",
"github.com/ethereum/go-ethereum/core/rawdb",
"github.com/ethereum/go-ethereum/core/types",
"github.com/ethereum/go-ethereum/crypto",
"github.com/ethereum/go-ethereum/ethclient",
"github.com/ethereum/go-ethereum/ethdb",
"github.com/ethereum/go-ethereum/p2p",
"github.com/ethereum/go-ethereum/p2p/discv5",
"github.com/ethereum/go-ethereum/rlp",
"github.com/ethereum/go-ethereum/rpc",
"github.com/hashicorp/golang-lru",
"github.com/hpcloud/tail",
"github.com/jmoiron/sqlx",
"github.com/lib/pq",
"github.com/mitchellh/go-homedir",
"github.com/onsi/ginkgo",
"github.com/onsi/gomega",
"github.com/onsi/gomega/ghttp",
"github.com/sirupsen/logrus",
"github.com/spf13/cobra",
"github.com/spf13/viper",
"golang.org/x/net/context",
"golang.org/x/sync/errgroup",
"gopkg.in/tomb.v1",
]
solver-name = "gps-cdcl"
solver-version = 1
-54
View File
@@ -1,54 +0,0 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
[[override]]
name = "gopkg.in/fsnotify.v1"
source = "gopkg.in/fsnotify/fsnotify.v1"
[[override]]
name = "github.com/pressly/sup"
version = "0.5.3"
[[constraint]]
name = "github.com/onsi/ginkgo"
version = "1.4.0"
[[constraint]]
branch = "master"
name = "github.com/jmoiron/sqlx"
[[constraint]]
branch = "master"
name = "github.com/lib/pq"
[[constraint]]
name = "github.com/sirupsen/logrus"
version = "1.2.0"
[[constraint]]
name = "github.com/spf13/cobra"
version = "0.0.1"
[[constraint]]
name = "github.com/ethereum/go-ethereum"
revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
+2 -11
View File
@@ -3,11 +3,6 @@ BASE = $(GOPATH)/src/$(PACKAGE)
PKGS = go list ./... | grep -v "^vendor/"
# Tools
## Dependency management
DEP = $(BIN)/dep
$(BIN)/dep:
go get -u github.com/golang/dep/cmd/dep
## Testing library
GINKGO = $(BIN)/ginkgo
$(BIN)/ginkgo:
@@ -32,7 +27,7 @@ $(BIN)/gometalinter.v2:
.PHONY: installtools
installtools: | $(LINT) $(GOOSE) $(GINKGO) $(DEP)
installtools: | $(LINT) $(GOOSE) $(GINKGO)
echo "Installing tools"
.PHONY: metalint
@@ -58,11 +53,7 @@ integrationtest: | $(GINKGO) $(LINT)
go fmt ./...
$(GINKGO) -r integration_test/
.PHONY: dep
dep: | $(DEP)
$(DEP) ensure
build: dep
build:
go fmt ./...
go build
+57 -43
View File
@@ -7,50 +7,56 @@
## Table of Contents
1. [Background](../staging/README.md#background)
1. [Dependencies](../staging/README.md#dependencies)
1. [Install](../staging/README.md#install)
1. [Usage](../staging/README.md#usage)
1. [Tests](../staging/README.md#tests)
1. [API](../staging/README.md#API)
1. [Contributing](../staging/README.md#contributing)
1. [License](../staging/README.md#license)
1. [Background](#background)
1. [Install](#install)
1. [Usage](#usage)
1. [Contributing](#contributing)
1. [License](#license)
## Background
The same data structures and encodings that make Ethereum an effective and trust-less distributed virtual machine
complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by
providing a suite of tools to ease the extraction and transformation of data into a more useful state.
complicate data accessibility and usability for dApp developers. VulcanizeDB improves Ethereum data accessibility by
providing a suite of tools to ease the extraction and transformation of data into a more useful state, including
allowing for exposing aggregate data from a suite of smart contracts.
VulanizeDB includes processes that sync, transform and expose data. Syncing involves
querying an Ethereum node and then persisting core data into a Postgres database. Transforming focuses on using previously synced data to
query for and transform log event and storage data for specifically configured smart contract addresses. Exposing data is a matter of getting
data from VulcanizeDB's underlying Postgres database and making it accessible.
## Dependencies
- Go 1.11+
- Postgres 10.6
![VulcanizeDB Overview Diagram](documentation/diagrams/vdb-overview.png)
## Install
1. [Dependencies](#dependencies)
1. [Building the project](#building-the-project)
1. [Setting up the database](#setting-up-the-database)
1. [Configuring a synced Ethereum node](#configuring-a-synced-ethereum-node)
### Dependencies
- Go 1.12+
- Postgres 11.2
- Ethereum Node
- [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.23+)
- [Parity 1.8.11+](https://github.com/paritytech/parity/releases)
## Install
1. [Building the project](../staging/README.md#building-the-project)
1. [Setting up the database](../staging/README.md#setting-up-the-database)
1. [Configuring a synced Ethereum node](../staging/README.md#configuring-a-synced-ethereum-node)
### Building the project
Download the codebase to your local `GOPATH` via:
`go get github.com/vulcanize/vulcanizedb`
Move to the project directory and use [golang/dep](https://github.com/golang/dep) to install the dependencies:
Move to the project directory:
`cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
`dep ensure`
Once the dependencies have been successfully installed, build the executable with:
Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:
`make build`
If you need to use a different dependency than what is currently defined in `go.mod`, it may helpful to look into [the replace directive](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
This instruction enables you to point at a fork or the local filesystem for dependency resolution.
If you are running into issues at this stage, ensure that `GOPATH` is defined in your shell.
If necessary, `GOPATH` can be set in `~/.bashrc` or `~/.bash_profile`, depending upon your system.
It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
@@ -68,7 +74,8 @@ It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
* See below for configuring additional environments
In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
In some cases (such as recent Ubuntu systems), it may be necessary to overcome failures of password authentication from
localhost. To allow access on Ubuntu, set localhost connections via hostname, ipv4, and ipv6 from peer/md5 to trust in: /etc/postgresql/<version>/pg_hba.conf
(It should be noted that trusted auth should only be enabled on systems without sensitive data in them: development and local test databases)
@@ -80,8 +87,9 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
- The IPC file is called `geth.ipc`.
- The geth IPC file path is printed to the console when you start geth.
- The default location is:
- Mac: `<full home path>/Library/Ethereum`
- Mac: `<full home path>/Library/Ethereum/geth.ipc`
- Linux: `<full home path>/ethereum/geth.ipc`
- Note: the geth.ipc file may not exist until you've started the geth process
- For Parity:
- The IPC file is called `jsonrpc.ipc`.
@@ -98,27 +106,35 @@ In some cases (such as recent Ubuntu systems), it may be necessary to overcome f
## Usage
Usage is broken up into two processes:
As mentioned above, VulcanizeDB's processes can be split into three categories: syncing, transforming and exposing data.
### Data syncing
To provide data for transformations, raw Ethereum data must first be synced into vDB.
This is accomplished through the use of the `headerSync`, `sync`, or `coldImport` commands.
These commands are described in detail [here](../staging/documentation/sync.md).
To provide data for transformations, raw Ethereum data must first be synced into VulcanizeDB.
This is accomplished through the use of the `headerSync`, `fullSync`, or `coldImport` commands.
These commands are described in detail [here](documentation/data-syncing.md).
### Data transformation
Contract watchers use the raw data that has been synced into Postgres to filter out and apply transformations to specific data of interest.
Data transformation uses the raw data that has been synced into Postgres to filter out and apply transformations to
specific data of interest. Since there are different types of data that may be useful for observing smart contracts, it
follows that there are different ways to transform this data. We've started by categorizing this into Generic and
Custom transformers:
There is a built-in `contractWatcher` command which provides generic transformation of most contract data.
The `contractWatcher` command is described further [here](../staging/documentation/contractWatcher.md).
- Generic Contract Transformer: Generic contract transformation can be done using a built-in command,
`contractWatcher`, which transforms contract events provided the contract's ABI is available. It also
provides some state variable coverage by automating polling of public methods, with some restrictions.
`contractWatcher` is described further [here](documentation/generic-transformer.md).
In many cases a custom transformer or set of transformers will need to be written to provide complete or more comprehensive coverage or to optimize other aspects of the output for a specific end-use.
In this case we have provided the `compose`, `execute`, and `composeAndExecute` commands for running custom transformers from external repositories.
- Custom Transformers: In many cases custom transformers will need to be written to provide
more comprehensive coverage of contract data. In this case we have provided the `compose`, `execute`, and
`composeAndExecute` commands for running custom transformers from external repositories. Documentation on how to write,
build and run custom transformers as Go plugins can be found
[here](documentation/custom-transformers.md).
Usage of the `compose`, `execute`, and `composeAndExecute` commands is described further [here](../staging/documentation/composeAndExecute.md).
### Exposing the data
[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](documentation/postgraphile.md).
Documentation on how to build custom transformers to work with these commands can be found [here](../staging/documentation/transformers.md).
## Tests
### Tests
- Replace the empty `ipcPath` in the `environments/infura.toml` with a path to a full node's eth_jsonrpc endpoint (e.g. local geth node ipc path or infura url)
- Note: integration tests require configuration with an archival node
- `createdb vulcanize_private` will create the test db
@@ -126,15 +142,13 @@ Documentation on how to build custom transformers to work with these commands ca
- `make test` will run the unit tests and skip the integration tests
- `make integrationtest` will run just the integration tests
## API
[Postgraphile](https://www.graphile.org/postgraphile/) is used to expose GraphQL endpoints for our database schemas, this is described in detail [here](../staging/postgraphile/README.md).
## Contributing
Contributions are welcome! For more on this, please see [here](../staging/documentation/contributing.md).
Contributions are welcome!
Small note: If editing the Readme, please conform to the [standard-readme specification](https://github.com/RichardLitt/standard-readme).
VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
For more information on contributing, please see [here](documentation/contributing.md).
## License
[AGPL-3.0](../staging/LICENSE) © Vulcanize Inc
[AGPL-3.0](LICENSE) © Vulcanize Inc
+8 -6
View File
@@ -39,6 +39,8 @@ var coldImportCmd = &cobra.Command{
Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
coldImport()
},
}
@@ -55,7 +57,7 @@ func coldImport() {
ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
ethDB, err := ethereum.CreateDatabase(ethDBConfig)
if err != nil {
log.Fatal("Error connecting to ethereum db: ", err)
LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
}
mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
if syncAll {
@@ -63,10 +65,10 @@ func coldImport() {
endingBlockNumber = mostRecentBlockNumberInDb
}
if endingBlockNumber < startingBlockNumber {
log.Fatal("Ending block number must be greater than starting block number for cold import.")
LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
}
if endingBlockNumber > mostRecentBlockNumberInDb {
log.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
}
// init pg db
@@ -76,13 +78,13 @@ func coldImport() {
nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
if err != nil {
log.Fatal("Error getting node: ", err)
LogWithCommand.Fatal("Error getting node: ", err)
}
pgDB := utils.LoadPostgres(databaseConfig, coldNode)
// init cold importer deps
blockRepository := repositories.NewBlockRepository(&pgDB)
receiptRepository := repositories.ReceiptRepository{DB: &pgDB}
receiptRepository := repositories.FullSyncReceiptRepository{DB: &pgDB}
transactionConverter := cold_db.NewColdDbTransactionConverter()
blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
@@ -90,6 +92,6 @@ func coldImport() {
coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
if err != nil {
log.Fatal("Error executing cold import: ", err)
LogWithCommand.Fatal("Error executing cold import: ", err)
}
}
+24 -19
View File
@@ -1,17 +1,18 @@
// Copyright © 2019 Vulcanize, Inc
//
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
@@ -101,6 +102,8 @@ single config file or in separate command instances using different config files
Specify config location when executing the command:
./vulcanizedb compose --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
compose()
},
}
@@ -110,23 +113,25 @@ func compose() {
prepConfig()
// Generate code to build the plugin according to the config file
log.Info("generating plugin")
LogWithCommand.Info("generating plugin")
generator, err := p2.NewGenerator(genConfig, databaseConfig)
if err != nil {
log.Fatal(err)
LogWithCommand.Debug("initializing plugin generator failed")
LogWithCommand.Fatal(err)
}
err = generator.GenerateExporterPlugin()
if err != nil {
log.Debug("generating plugin failed")
log.Fatal(err)
LogWithCommand.Debug("generating plugin failed")
LogWithCommand.Fatal(err)
}
// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
log.Fatal(err)
LogWithCommand.Debug("getting plugin path failed")
LogWithCommand.Fatal(err)
}
fmt.Printf("Composed plugin %s", pluginPath)
log.Info("plugin .so file output to", pluginPath)
LogWithCommand.Info("plugin .so file output to ", pluginPath)
}
func init() {
@@ -134,38 +139,38 @@ func init() {
}
func prepConfig() {
log.Info("configuring plugin")
LogWithCommand.Info("configuring plugin")
names := viper.GetStringSlice("exporter.transformerNames")
transformers := make(map[string]config.Transformer)
for _, name := range names {
transformer := viper.GetStringMapString("exporter." + name)
p, pOK := transformer["path"]
if !pOK || p == "" {
log.Fatal(name, "transformer config is missing `path` value")
LogWithCommand.Fatal(name, " transformer config is missing `path` value")
}
r, rOK := transformer["repository"]
if !rOK || r == "" {
log.Fatal(name, "transformer config is missing `repository` value")
LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
}
m, mOK := transformer["migrations"]
if !mOK || m == "" {
log.Fatal(name, "transformer config is missing `migrations` value")
LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
}
mr, mrOK := transformer["rank"]
if !mrOK || mr == "" {
log.Fatal(name, "transformer config is missing `rank` value")
LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
}
rank, err := strconv.ParseUint(mr, 10, 64)
if err != nil {
log.Fatal(name, "migration `rank` can't be converted to an unsigned integer")
LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
}
t, tOK := transformer["type"]
if !tOK {
log.Fatal(name, "transformer config is missing `type` value")
LogWithCommand.Fatal(name, " transformer config is missing `type` value")
}
transformerType := config.GetTransformerType(t)
if transformerType == config.UnknownTransformerType {
log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
}
transformers[name] = config.Transformer{
+21 -18
View File
@@ -1,17 +1,18 @@
// Copyright © 2019 Vulcanize, Inc
//
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
@@ -105,6 +106,8 @@ single config file or in separate command instances using different config files
Specify config location when executing the command:
./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
composeAndExecute()
},
}
@@ -114,44 +117,44 @@ func composeAndExecute() {
prepConfig()
// Generate code to build the plugin according to the config file
log.Info("generating plugin")
LogWithCommand.Info("generating plugin")
generator, err := p2.NewGenerator(genConfig, databaseConfig)
if err != nil {
log.Fatal(err)
LogWithCommand.Fatal(err)
}
err = generator.GenerateExporterPlugin()
if err != nil {
log.Debug("generating plugin failed")
log.Fatal(err)
LogWithCommand.Debug("generating plugin failed")
LogWithCommand.Fatal(err)
}
// Get the plugin path and load the plugin
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
log.Fatal(err)
LogWithCommand.Fatal(err)
}
if !genConfig.Save {
defer helpers.ClearFiles(pluginPath)
}
log.Info("linking plugin", pluginPath)
LogWithCommand.Info("linking plugin ", pluginPath)
plug, err := plugin.Open(pluginPath)
if err != nil {
log.Debug("linking plugin failed")
log.Fatal(err)
LogWithCommand.Debug("linking plugin failed")
LogWithCommand.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
log.Info("loading transformers from plugin")
LogWithCommand.Info("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter")
if err != nil {
log.Debug("loading Exporter symbol failed")
log.Fatal(err)
LogWithCommand.Debug("loading Exporter symbol failed")
LogWithCommand.Fatal(err)
}
// Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter)
if !ok {
log.Debug("plugged-in symbol not of type Exporter")
LogWithCommand.Debug("plugged-in symbol not of type Exporter")
os.Exit(1)
}
@@ -193,5 +196,5 @@ func composeAndExecute() {
func init() {
rootCmd.AddCommand(composeAndExecuteCmd)
composeAndExecuteCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
composeAndExecuteCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
}
+7 -5
View File
@@ -26,7 +26,7 @@ import (
st "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
ft "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
lt "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
ht "github.com/vulcanize/vulcanizedb/pkg/contract_watcher/header/transformer"
"github.com/vulcanize/vulcanizedb/utils"
)
@@ -79,6 +79,8 @@ Requires a .toml config file:
piping = true
`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
contractWatcher()
},
}
@@ -99,22 +101,22 @@ func contractWatcher() {
con.PrepConfig()
switch mode {
case "header":
t = lt.NewTransformer(con, blockChain, &db)
t = ht.NewTransformer(con, blockChain, &db)
case "full":
t = ft.NewTransformer(con, blockChain, &db)
default:
log.Fatal("Invalid mode")
LogWithCommand.Fatal("Invalid mode")
}
err := t.Init()
if err != nil {
log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
}
for range ticker.C {
err = t.Execute()
if err != nil {
log.Error("Execution error for transformer:", t.GetConfig().Name, err)
LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
}
}
}
+20 -19
View File
@@ -1,23 +1,23 @@
// Copyright © 2019 Vulcanize, Inc
//
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"os"
"plugin"
syn "sync"
"time"
@@ -64,6 +64,8 @@ must have been composed by the same version of vulcanizedb or else it will not b
Specify config location when executing the command:
./vulcanizedb execute --config=./environments/config_name.toml`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
execute()
},
}
@@ -75,30 +77,29 @@ func execute() {
// Get the plugin path and load the plugin
_, pluginPath, err := genConfig.GetPluginPaths()
if err != nil {
log.Fatal(err)
LogWithCommand.Fatal(err)
}
fmt.Printf("Executing plugin %s", pluginPath)
log.Info("linking plugin", pluginPath)
LogWithCommand.Info("linking plugin ", pluginPath)
plug, err := plugin.Open(pluginPath)
if err != nil {
log.Debug("linking plugin failed")
log.Fatal(err)
LogWithCommand.Warn("linking plugin failed")
LogWithCommand.Fatal(err)
}
// Load the `Exporter` symbol from the plugin
log.Info("loading transformers from plugin")
LogWithCommand.Info("loading transformers from plugin")
symExporter, err := plug.Lookup("Exporter")
if err != nil {
log.Debug("loading Exporter symbol failed")
log.Fatal(err)
LogWithCommand.Warn("loading Exporter symbol failed")
LogWithCommand.Fatal(err)
}
// Assert that the symbol is of type Exporter
exporter, ok := symExporter.(Exporter)
if !ok {
log.Debug("plugged-in symbol not of type Exporter")
os.Exit(1)
LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
}
// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
@@ -139,7 +140,7 @@ func execute() {
func init() {
rootCmd.AddCommand(executeCmd)
executeCmd.Flags().BoolVarP(&recheckHeadersArg, "recheck-headers", "r", false, "whether to re-check headers for watched events")
executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "how often to recheck queued storage diffs")
executeCmd.Flags().DurationVarP(&queueRecheckInterval, "queue-recheck-interval", "q", 5*time.Minute, "interval duration for rechecking queued storage diffs (ex: 5m30s)")
}
type Exporter interface {
@@ -149,7 +150,7 @@ type Exporter interface {
func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the EventTransformerInitializer set using the watcher
log.Info("executing event transformers")
LogWithCommand.Info("executing event transformers")
var recheck constants.TransformerExecution
if recheckHeadersArg {
recheck = constants.HeaderRecheck
@@ -166,7 +167,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the StorageTransformerInitializer set using the storage watcher
log.Info("executing storage transformers")
LogWithCommand.Info("executing storage transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
@@ -179,7 +180,7 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
defer wg.Done()
// Execute over the ContractTransformerInitializer set using the contract watcher
log.Info("executing contract_watcher transformers")
LogWithCommand.Info("executing contract_watcher transformers")
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
for range ticker.C {
+16 -15
View File
@@ -29,14 +29,14 @@ import (
"github.com/vulcanize/vulcanizedb/utils"
)
// syncCmd represents the sync command
var syncCmd = &cobra.Command{
Use: "sync",
// fullSyncCmd represents the fullSync command
var fullSyncCmd = &cobra.Command{
Use: "fullSync",
Short: "Syncs VulcanizeDB with local ethereum node",
Long: `Syncs VulcanizeDB with local ethereum node. Populates
Postgres with blocks, transactions, receipts, and logs.
./vulcanizedb sync --starting-block-number 0 --config public.toml
./vulcanizedb fullSync --starting-block-number 0 --config public.toml
Expects ethereum node to be running and requires a .toml config:
@@ -49,38 +49,39 @@ Expects ethereum node to be running and requires a .toml config:
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
sync()
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
fullSync()
},
}
func init() {
rootCmd.AddCommand(syncCmd)
syncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
rootCmd.AddCommand(fullSyncCmd)
fullSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
}
func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
if err != nil {
log.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
}
missingBlocksPopulated <- populated
}
func sync() {
func fullSync() {
ticker := time.NewTicker(pollingInterval)
defer ticker.Stop()
blockChain := getBlockChain()
lastBlock, err := blockChain.LastBlock()
if err != nil {
log.Error("sync: Error getting last block: ", err)
LogWithCommand.Error("fullSync: Error getting last block: ", err)
}
if lastBlock.Int64() == 0 {
log.Fatal("geth initial: state sync not finished")
LogWithCommand.Fatal("geth initial: state sync not finished")
}
if startingBlockNumber > lastBlock.Int64() {
log.Fatal("sync: starting block number > current block number")
LogWithCommand.Fatal("fullSync: starting block number > current block number")
}
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
@@ -94,9 +95,9 @@ func sync() {
case <-ticker.C:
window, err := validator.ValidateBlocks()
if err != nil {
log.Error("sync: error in validateBlocks: ", err)
LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
}
log.Info(window.GetString())
LogWithCommand.Debug(window.GetString())
case <-missingBlocksPopulated:
go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
}
+8 -6
View File
@@ -50,6 +50,8 @@ Expects ethereum node to be running and requires a .toml config:
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
`,
Run: func(cmd *cobra.Command, args []string) {
SubCommand = cmd.CalledAs()
LogWithCommand = *log.WithField("SubCommand", SubCommand)
headerSync()
},
}
@@ -64,7 +66,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
if err != nil {
// TODO Lots of possible errors in the call stack above. If errors occur, we still put
// 0 in the channel, triggering another round
log.Error("backfillAllHeaders: Error populating headers: ", err)
LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
}
missingBlocksPopulated <- populated
}
@@ -86,9 +88,9 @@ func headerSync() {
case <-ticker.C:
window, err := validator.ValidateHeaders()
if err != nil {
log.Error("headerSync: ValidateHeaders failed: ", err)
LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
}
log.Info(window.GetString())
LogWithCommand.Debug(window.GetString())
case n := <-missingBlocksPopulated:
if n == 0 {
time.Sleep(3 * time.Second)
@@ -101,12 +103,12 @@ func headerSync() {
func validateArgs(blockChain *geth.BlockChain) {
lastBlock, err := blockChain.LastBlock()
if err != nil {
log.Error("validateArgs: Error getting last block: ", err)
LogWithCommand.Error("validateArgs: Error getting last block: ", err)
}
if lastBlock.Int64() == 0 {
log.Fatal("geth initial: state sync not finished")
LogWithCommand.Fatal("geth initial: state sync not finished")
}
if startingBlockNumber > lastBlock.Int64() {
log.Fatal("starting block number > current block number")
LogWithCommand.Fatal("starting block number > current block number")
}
}
+31 -9
View File
@@ -18,7 +18,6 @@ package cmd
import (
"fmt"
"os"
"strings"
"time"
@@ -47,6 +46,8 @@ var (
syncAll bool
endingBlockNumber int64
recheckHeadersArg bool
SubCommand string
LogWithCommand log.Entry
)
const (
@@ -56,18 +57,26 @@ const (
var rootCmd = &cobra.Command{
Use: "vulcanizedb",
PersistentPreRun: database,
PersistentPreRun: initFuncs,
}
func Execute() {
log.Info("----- Starting vDB -----")
if err := rootCmd.Execute(); err != nil {
log.Fatal(err)
os.Exit(1)
}
}
func database(cmd *cobra.Command, args []string) {
func initFuncs(cmd *cobra.Command, args []string) {
setViperConfigs()
logLvlErr := logLevel()
if logLvlErr != nil {
log.Fatal("Could not set log level: ", logLvlErr)
}
}
func setViperConfigs() {
ipc = viper.GetString("client.ipcpath")
levelDbPath = viper.GetString("client.leveldbpath")
storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
@@ -81,6 +90,19 @@ func database(cmd *cobra.Command, args []string) {
viper.Set("database.config", databaseConfig)
}
func logLevel() error {
lvl, err := log.ParseLevel(viper.GetString("log.level"))
if err != nil {
return err
}
log.SetLevel(lvl)
if lvl > log.InfoLevel {
log.SetReportCaller(true)
}
log.Info("Log level set to ", lvl.String())
return nil
}
func init() {
cobra.OnInitialize(initConfig)
// When searching for env variables, replace dots in config keys with underscores
@@ -97,6 +119,7 @@ func init() {
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@@ -107,6 +130,7 @@ func init() {
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
}
func initConfig() {
@@ -116,16 +140,14 @@ func initConfig() {
noConfigError := "No config file passed with --config flag"
fmt.Println("Error: ", noConfigError)
log.Fatal(noConfigError)
os.Exit(1)
}
if err := viper.ReadInConfig(); err == nil {
log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
} else {
invalidConfigError := "Couldn't read config file"
fmt.Println("Error: ", invalidConfigError)
log.Fatal(invalidConfigError)
os.Exit(1)
formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error())
log.Fatal(formattedError)
}
}
@@ -133,7 +155,7 @@ func getBlockChain() *geth.BlockChain {
rawRpcClient, err := rpc.Dial(ipc)
if err != nil {
log.Fatal(err)
LogWithCommand.Fatal(err)
}
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
ethClient := ethclient.NewClient(rawRpcClient)
+10
View File
@@ -0,0 +1,10 @@
-- +goose Up
CREATE TABLE public.addresses
(
id SERIAL PRIMARY KEY,
address character varying(42),
UNIQUE (address)
);
-- +goose Down
DROP TABLE public.addresses;
@@ -1,16 +0,0 @@
-- +goose Up
CREATE TABLE full_sync_receipts
(
id SERIAL PRIMARY KEY,
transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
contract_address VARCHAR(42),
cumulative_gas_used NUMERIC,
gas_used NUMERIC,
state_root VARCHAR(66),
status INTEGER,
tx_hash VARCHAR(66)
);
-- +goose Down
DROP TABLE full_sync_receipts;
@@ -0,0 +1,16 @@
-- +goose Up
CREATE TABLE full_sync_receipts
(
id SERIAL PRIMARY KEY,
transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
cumulative_gas_used NUMERIC,
gas_used NUMERIC,
state_root VARCHAR(66),
status INTEGER,
tx_hash VARCHAR(66)
);
-- +goose Down
DROP TABLE full_sync_receipts;
@@ -5,7 +5,8 @@ CREATE TABLE public.queued_storage (
block_hash BYTEA,
contract BYTEA,
storage_key BYTEA,
storage_value BYTEA
storage_value BYTEA,
UNIQUE (block_height, block_hash, contract, storage_key, storage_value)
);
-- +goose Down
@@ -1,18 +0,0 @@
-- +goose Up
CREATE TABLE header_sync_receipts(
id SERIAL PRIMARY KEY,
transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE,
header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
contract_address VARCHAR(42),
cumulative_gas_used NUMERIC,
gas_used NUMERIC,
state_root VARCHAR(66),
status INTEGER,
tx_hash VARCHAR(66),
rlp BYTEA,
UNIQUE(header_id, transaction_id)
);
-- +goose Down
DROP TABLE header_sync_receipts;
@@ -0,0 +1,19 @@
-- +goose Up
CREATE TABLE header_sync_receipts
(
id SERIAL PRIMARY KEY,
transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions (id) ON DELETE CASCADE,
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
cumulative_gas_used NUMERIC,
gas_used NUMERIC,
state_root VARCHAR(66),
status INTEGER,
tx_hash VARCHAR(66),
rlp BYTEA,
UNIQUE (header_id, transaction_id)
);
-- +goose Down
DROP TABLE header_sync_receipts;
+82 -18
View File
@@ -2,8 +2,8 @@
-- PostgreSQL database dump
--
-- Dumped from database version 10.6
-- Dumped by pg_dump version 10.6
-- Dumped from database version 11.3
-- Dumped by pg_dump version 11.3
SET statement_timeout = 0;
SET lock_timeout = 0;
@@ -12,27 +12,44 @@ SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
--
-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
--
CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
--
-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
--
COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
SET default_tablespace = '';
SET default_with_oids = false;
--
-- Name: addresses; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.addresses (
id integer NOT NULL,
address character varying(42)
);
--
-- Name: addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.addresses_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id;
--
-- Name: logs; Type: TABLE; Schema: public; Owner: -
--
@@ -157,7 +174,7 @@ CREATE TABLE public.eth_nodes (
CREATE TABLE public.full_sync_receipts (
id integer NOT NULL,
contract_address character varying(42),
contract_address_id integer NOT NULL,
cumulative_gas_used numeric,
gas_used numeric,
state_root character varying(66),
@@ -267,7 +284,7 @@ CREATE TABLE public.header_sync_receipts (
id integer NOT NULL,
transaction_id integer NOT NULL,
header_id integer NOT NULL,
contract_address character varying(42),
contract_address_id integer NOT NULL,
cumulative_gas_used numeric,
gas_used numeric,
state_root character varying(66),
@@ -577,6 +594,13 @@ CREATE VIEW public.watched_event_logs AS
WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
--
-- Name: addresses id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass);
--
-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -675,6 +699,22 @@ ALTER TABLE ONLY public.uncles ALTER COLUMN id SET DEFAULT nextval('public.uncle
ALTER TABLE ONLY public.watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('public.watched_contracts_contract_id_seq'::regclass);
--
-- Name: addresses addresses_address_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.addresses
ADD CONSTRAINT addresses_address_key UNIQUE (address);
--
-- Name: addresses addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.addresses
ADD CONSTRAINT addresses_pkey PRIMARY KEY (id);
--
-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -803,6 +843,14 @@ ALTER TABLE ONLY public.eth_nodes
ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
--
-- Name: queued_storage queued_storage_block_height_block_hash_contract_storage_key_key; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.queued_storage
ADD CONSTRAINT queued_storage_block_height_block_hash_contract_storage_key_key UNIQUE (block_height, block_hash, contract, storage_key, storage_value);
--
-- Name: queued_storage queued_storage_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -901,6 +949,14 @@ ALTER TABLE ONLY public.checked_headers
ADD CONSTRAINT checked_headers_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
--
-- Name: full_sync_receipts full_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.full_sync_receipts
ADD CONSTRAINT full_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
--
-- Name: full_sync_transactions full_sync_transactions_block_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -909,6 +965,14 @@ ALTER TABLE ONLY public.full_sync_transactions
ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
--
-- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.header_sync_receipts
ADD CONSTRAINT header_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
--
-- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
--
+42 -3
View File
@@ -1,11 +1,50 @@
# Contribution guidelines
Contributions are welcome! In addition to core contributions, developers are encouraged to build their own custom transformers which
can be run together with other custom transformers using the [composeAndExeucte](../../staging/documentation/composeAndExecute.md) command.
Contributions are welcome! Please open an Issues or Pull Request for any changes.
In addition to core contributions, developers are encouraged to build their own custom transformers which
can be run together with other custom transformers using the [composeAndExecute](../../staging/documentation/custom-transformers.md) command.
## Pull Requests
- `go fmt` is run as part of `make test` and `make integrationtest`, please make sure to check in the format changes.
- Ensure that new code is well tested, including integration testing if applicable.
- Make sure the build is passing.
- Update the README or any [documentation files](./) as necessary. If editing the Readme, please
conform to the
[standard-readme specification](https://github.com/RichardLitt/standard-readme).
- Once a Pull Request has received two approvals it can be merged in by a core developer.
## Creating a new migration file
1. `make new_migration NAME=add_columnA_to_table1`
- This will create a new timestamped migration file in `db/migrations`
1. Write the migration code in the created file, under the respective `goose` pragma
- Goose automatically runs each migration in a transaction; don't add `BEGIN` and `COMMIT` statements.
1. Core migrations should be committed in their `goose fix`ed form.
1. Core migrations should be committed in their `goose fix`ed form. To do this, run `make version_migrations` which
converts timestamped migrations to migrations versioned by an incremented integer.
## Diagrams
- Diagrams were created with [draw.io](https://www.draw.io).
- To update a diagram:
1. Go to [draw.io](https://www.draw.io).
1. Click on *File > Open from* and choose the location of the diagram you want to update.
1. Once open in draw.io, you may update it.
1. Export the diagram to this repository's directory and commit it.
## Generating the Changelog
We use [github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) to generate release Changelogs. To be consistent with previous Changelogs, the following flags should be passed to the command:
```
--user vulcanize
--project vulcanizedb
--token {YOUR_GITHUB_TOKEN}
--no-issues
--usernames-as-github-logins
--since-tag {PREVIOUS_RELEASE_TAG}
```
For more information on why your github token is needed, and how to generate it see [https://github
.com/github-changelog-generator/github-changelog-generator#github-token](https://github.com/github-changelog-generator/github-changelog-generator#github-token).
## Code of Conduct
VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
@@ -1,44 +1,80 @@
# composeAndExecute
The `composeAndExecute` command is used to compose and execute over an arbitrary set of custom transformers.
This is accomplished by generating a Go pluggin which allows the `vulcanizedb` binary to link to external transformers, so
long as they abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
# Custom Transformers
When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
purpose can be leveraged.
Additionally, there are separate `compose` and `execute` commands to allow pre-building and linking to a pre-built .so file.
Individual custom transformers can be composed together from any number of external repositories and executed as a
single process using the `compose` and `execute` commands or the `composeAndExecute` command. This is accomplished by
generating a Go plugin which allows the `vulcanizedb` binary to link to the external transformers, so long as they
abide by one of the standard [interfaces](../staging/libraries/shared/transformer).
**NOTE:**
1. It is necessary that the .so file was built with the same exact dependencies that are present in the execution environment,
i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
1. The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so in a different
environment than the one it was composed in then the migrations for that plugin will first need to be manually ran against that environment's Postgres database.
## Writing custom transformers
For help with writing different types of custom transformers please see below:
These commands require Go 1.11+ and use [Go plugins](https://golang.org/pkg/plugin/) which only work on Unix-based systems.
There is also an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use vendored dependencies which
imposes certain limitations on how the plugins are built.
Storage Transformers: transform data derived from contract storage tries
* [Guide](../../staging/libraries/shared/factories/storage/README.md)
* [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
## Commands
The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's `$GOPATH`,
and that all of the transformer repositories for building the plugin are present at their `$GOPATH` directories.
Event Transformers: transform data derived from Ethereum log events
* [Guide](../../staging/libraries/shared/factories/event/README.md)
* [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
* [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
* [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
The `execute` command does not require the plugin transformer dependencies be located in their
`$GOPATH` directories, instead it expects a prebuilt .so file (of the name specified in the config file)
to be in `$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin
db migrations to have already been ran against the database.
Contract Transformers: transform data derived from Ethereum log events and use it to poll public contract methods
* [Example 1](https://github.com/vulcanize/account_transformers)
* [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
compose:
## Preparing custom transformers to work as part of a plugin
To plug in an external transformer we need to:
`./vulcanizedb compose --config=./environments/config_name.toml`
1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
* Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
* Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
* If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
as they are [here](../../staging/db/migrations)
execute:
To update a plugin repository with changes to the core vulcanizedb repository, use your dependency manager to install the desired version of vDB.
`./vulcanizedb execute --config=./environments/config_name.toml`
## Building and Running Custom Transformers
### Commands
* The `compose`, `execute`, `composeAndExecute` commands require Go 1.11+ and use [Go plugins](https://golang
.org/pkg/plugin/) which only work on Unix-based systems.
composeAndExecute:
* There is an ongoing [conflict](https://github.com/golang/go/issues/20481) between Go plugins and the use of vendored
dependencies which imposes certain limitations on how the plugins are built.
`./vulcanizedb composeAndExecute --config=./environments/config_name.toml`
* Separate `compose` and `execute` commands allow pre-building and linking to the pre-built .so file. So, if
these are run independently, instead of using `composeAndExecute`, a couple of things need to be considered:
* It is necessary that the .so file was built with the same exact dependencies that are present in the execution
environment, i.e. we need to `compose` and `execute` the plugin .so file with the same exact version of vulcanizeDB.
* The plugin migrations are run during the plugin's composition. As such, if `execute` is used to run a prebuilt .so
in a different environment than the one it was composed in, then the database structure will need to be loaded
into the environment's Postgres database. This can either be done by manually loading the plugin's schema into
Postgres, or by manually running the plugin's migrations.
* The `compose` and `composeAndExecute` commands assume you are in the vulcanizdb directory located at your system's
`$GOPATH`, and that the plugin dependencies are present at their `$GOPATH` directories.
## Flags
* The `execute` command does not require the plugin transformer dependencies be located in their `$GOPATH` directories,
instead it expects a .so file (of the name specified in the config file) to be in
`$GOPATH/src/github.com/vulcanize/vulcanizedb/plugins/` and, as noted above, also expects the plugin db migrations to
have already been ran against the database.
The `compose` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
* Usage:
* compose: `./vulcanizedb compose --config=environments/config_name.toml`
* execute: `./vulcanizedb execute --config=environments/config_name.toml`
* composeAndExecute: `./vulcanizedb composeAndExecute --config=environments/config_name.toml`
### Flags
The `execute` and `composeAndExecute` commands can be passed optional flags to specify the operation of the watchers:
- `--recheck-headers`/`-r` - specifies whether to re-check headers for events after the header has already been queried for watched logs.
Can be useful for redundancy if you suspect that your node is not always returning all desired logs on every query.
@@ -50,7 +86,7 @@ Defaults to `false`.
Argument is expected to be a duration (integer measured in nanoseconds): e.g. `-q=10m30s` (for 10 minute, 30 second intervals).
Defaults to `5m` (5 minutes).
## Configuration
### Configuration
A .toml config file is specified when executing the commands.
The config provides information for composing a set of transformers from external repositories:
+81
View File
@@ -0,0 +1,81 @@
# Syncing commands
These commands are used to sync raw Ethereum data into Postgres, with varying levels of data granularity.
## headerSync
Syncs block headers from a running Ethereum node into the VulcanizeDB table `headers`.
- Queries the Ethereum node using RPC calls.
- Validates headers from the last 15 blocks to ensure that data is up to date.
- Useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart
contract storage values or event logs).
- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
different from what we have already stored in the database, the header record will be updated.
#### Usage
- Run: `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
```toml
[database]
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
password = "vulcanize"
port = 5432
[client]
ipcPath = <path to a running Ethereum node>
```
- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
## fullSync
Syncs blocks, transactions, receipts and logs from a running Ethereum node into VulcanizeDB tables named
`blocks`, `uncles`, `full_sync_transactions`, `full_sync_receipts` and `logs`.
- Queries the Ethereum node using RPC calls.
- Validates headers from the last 15 blocks to ensure that data is up to date.
- Useful when you want to maintain a broad cache of what's happening on the blockchain.
- Handles chain reorgs by [validating the most recent blocks' hashes](../pkg/history/header_validator.go). If the hash is
different from what we have already stored in the database, the header record will be updated.
#### Usage
- Run `./vulcanizedb fullSync --config <config.toml> --starting-block-number <block-number>`
- The config file must be formatted as follows, and should contain an ipc path to a running Ethereum node:
```toml
[database]
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
password = "vulcanize"
port = 5432
[client]
ipcPath = <path to a running Ethereum node>
```
- Alternatively, the ipc path can be passed as a flag instead `--client-ipcPath`.
*Please note, that if you are fast syncing your Ethereum node, wait for the initial sync to finish.*
## coldImport
Syncs VulcanizeDB from Geth's underlying LevelDB datastore and persists Ethereum blocks,
transactions, receipts and logs into VulcanizeDB tables named `blocks`, `uncles`,
`full_sync_transactions`, `full_sync_receipts` and `logs` respectively.
#### Usage
1. Ensure the Ethereum node you're point at is not running, and that it has synced to the desired block height.
1. Run `./vulcanizedb coldImport --config <config.toml>`
1. Optional flags:
- `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
- `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
- `--all`/`-a`: sync all missing blocks
The config file can be formatted as follows, and must contain the LevelDB path.
```toml
[database]
name = "vulcanize_public"
hostname = "localhost"
user = "vulcanize"
password = "vulcanize"
port = 5432
[client]
leveldbpath = "/Users/user/Library/Ethereum/geth/chaindata"
```
Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

@@ -1,4 +1,4 @@
# contractWatcher
# Generic Transformer
The `contractWatcher` command is a built-in generic contract watcher. It can watch any and all events for a given contract provided the contract's ABI is available.
It also provides some state variable coverage by automating polling of public methods, with some restrictions:
1. The method must have 2 or less arguments
+34
View File
@@ -0,0 +1,34 @@
# Postgraphile
You can expose VulcanizeDB data via [Postgraphile](https://github.com/graphile/postgraphile).
Check out [their documentation](https://www.graphile.org/postgraphile/) for the most up-to-date instructions on installing, running, and customizing Postgraphile.
## Simple Setup
As of April 30, 2019, you can run Postgraphile pointed at the default `vulcanize_public` database with the following commands:
```
npm install -g postgraphile
postgraphile --connection postgres://localhost/vulcanize_public --schema=public,custom --disable-default-mutations --no-ignore-rbac
```
Arguments:
- `--connection` specifies the database. The above command connects to the default `vulcanize_public` database
defined in [the example config](../environments/public.toml.example).
- `--schema` defines what schema(s) to expose. The above exposes the `public` schema (for core VulcanizeDB data) as well as a `custom` schema (where `custom` is the name of a schema defined in executed transformers).
- `--disable-default-mutations` prevents Postgraphile from exposing create, update, and delete operations on your data, which are otherwise enabled by default.
- `--no-ignore-rbac` ensures that Postgraphile will only expose the tables, columns, fields, and query functions that the user has explicit access to.
## Customizing Postgraphile
By default, Postgraphile will expose queries for all tables defined in your chosen database/schema(s), including [filtering](https://www.graphile.org/postgraphile/filtering/) and [auto-discovered relations](https://www.graphile.org/postgraphile/relations/).
If you'd like to expose more customized windows into your data, there are some techniques you can apply when writing migrations:
- [Computed columns](https://www.graphile.org/postgraphile/computed-columns/) enable you to derive additional fields from types defined in your database.
For example, you could write a function to expose a block header's state root over Postgraphile with a computed column - without modifying the `public.headers` table.
- [Custom queries](https://www.graphile.org/postgraphile/custom-queries/) enable you to provide on-demand access to more complex data (e.g. the product of joining and filtering several tables' data based on a passed argument).
For example, you could write a custom query to get the block timestamp for every transaction originating from a given address.
- [Subscriptions](https://www.graphile.org/postgraphile/subscriptions/) enable you to publish data as it is coming into your database.
The above list is not exhaustive - please see the Postgraphile documentation for a more comprehensive and up-to-date description of available features.
-26
View File
@@ -1,26 +0,0 @@
# Syncing commands
These commands are used to sync raw Ethereum data into Postgres.
## headerSync
Syncs VulcanizeDB with the configured Ethereum node, populating only block headers.
This command is useful when you want a minimal baseline from which to track targeted data on the blockchain (e.g. individual smart contract storage values or event logs).
1. Start Ethereum node
1. In a separate terminal start VulcanizeDB:
- `./vulcanizedb headerSync --config <config.toml> --starting-block-number <block-number>`
## sync
Syncs VulcanizeDB with the configured Ethereum node, populating blocks, transactions, receipts, and logs.
This command is useful when you want to maintain a broad cache of what's happening on the blockchain.
1. Start Ethereum node (**if fast syncing your Ethereum node, wait for initial sync to finish**)
1. In a separate terminal start VulcanizeDB:
- `./vulcanizedb sync --config <config.toml> --starting-block-number <block-number>`
## coldImport
Sync VulcanizeDB from the LevelDB underlying a Geth node.
1. Assure node is not running, and that it has synced to the desired block height.
1. Start vulcanize_db
- `./vulcanizedb coldImport --config <config.toml>`
1. Optional flags:
- `--starting-block-number <block number>`/`-s <block number>`: block number to start syncing from
- `--ending-block-number <block number>`/`-e <block number>`: block number to sync to
- `--all`/`-a`: sync all missing blocks
-40
View File
@@ -1,40 +0,0 @@
# Custom transformers
When the capabilities of the generic `contractWatcher` are not sufficient, custom transformers tailored to a specific
purpose can be leveraged.
Individual transformers can be composed together from any number of external repositories and executed as a single process using
the `compose` and `execute` commands or the `composeAndExecute` command.
## Writing custom transformers
For help with writing different types of custom transformers for the `composeAndExecute` set of commands, please see the below:
Storage Transformers
* [Guide](../../staging/libraries/shared/factories/storage/README.md)
* [Example](../../staging/libraries/shared/factories/storage/EXAMPLE.md)
Event Transformers
* [Guide](../../staging/libraries/shared/factories/event/README.md)
* [Example 1](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registar)
* [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/registry)
* [Example 3](https://github.com/vulcanize/ens_transformers/tree/master/transformers/resolver)
Contract Transformers
* [Example 1](https://github.com/vulcanize/account_transformers)
* [Example 2](https://github.com/vulcanize/ens_transformers/tree/master/transformers/domain_records)
## Preparing custom transformers to work as part of a plugin
To plug in an external transformer we need to:
1. Create a package that exports a variable `TransformerInitializer`, `StorageTransformerInitializer`, or `ContractTransformerInitializer` that are of type [TransformerInitializer](../staging/libraries/shared/transformer/event_transformer.go#L33)
or [StorageTransformerInitializer](../../staging/libraries/shared/transformer/storage_transformer.go#L31),
or [ContractTransformerInitializer](../../staging/libraries/shared/transformer/contract_transformer.go#L31), respectively
2. Design the transformers to work in the context of their [event](../staging/libraries/shared/watcher/event_watcher.go#L83),
[storage](../../staging/libraries/shared/watcher/storage_watcher.go#L53),
or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) watcher execution modes
3. Create db migrations to run against vulcanizeDB so that we can store the transformer output
* Do not `goose fix` the transformer migrations, this is to ensure they are always ran after the core vulcanizedb migrations which are kept in their fixed form
* Specify migration locations for each transformer in the config with the `exporter.transformer.migrations` fields
* If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
as they are [here](../../staging/db/migrations)
To update a plugin repository with changes to the core vulcanizedb repository, run `dep ensure` to update its dependencies.
+33
View File
@@ -0,0 +1,33 @@
module github.com/vulcanize/vulcanizedb
go 1.12
require (
github.com/allegro/bigcache v1.2.1 // indirect
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 // indirect
github.com/dave/jennifer v1.3.0
github.com/ethereum/go-ethereum v1.9.1
github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 // indirect
github.com/hashicorp/golang-lru v0.5.1
github.com/hpcloud/tail v1.0.0
github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a
github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 // indirect
github.com/lib/pq v1.0.0
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mitchellh/go-homedir v1.1.0
github.com/onsi/ginkgo v1.7.0
github.com/onsi/gomega v1.4.3
github.com/pborman/uuid v1.2.0 // indirect
github.com/pressly/goose v2.6.0+incompatible
github.com/prometheus/tsdb v0.10.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/sirupsen/logrus v1.2.0
github.com/spf13/cobra v0.0.3
github.com/spf13/viper v1.3.2
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 // indirect
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
github.com/tyler-smith/go-bip39 v1.0.2 // indirect
golang.org/x/net v0.0.0-20190603091049-60506f45cf65
golang.org/x/sync v0.0.0-20190423024810-112230192c58
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
)
+296
View File
@@ -0,0 +1,296 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc=
github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A=
github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU=
github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/aristanetworks/goarista v0.0.0-20190121184617-8f049bdb8feb/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw=
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo=
github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
github.com/ethereum/go-ethereum v1.9.0/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
github.com/ethereum/go-ethereum v1.9.1 h1:MrdTRvKIa3apdx6NW1azzSgl8BQB1eTBVSUmFhuztaU=
github.com/ethereum/go-ethereum v1.9.1/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 h1:9WiNlI9Cds5S5YITwRpRs8edNaq0nxTEymhDW20A1QE=
github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb v1.7.7 h1:UvNzAPfBrKMENVbQ4mr4ccA9sW+W1Ihl0Yh1s0BiVAg=
github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw=
github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68=
github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo=
github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.9.1/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic=
github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM=
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 h1:GnOzE5fEFN3b2zDhJJABEofdb51uMRNb8eqIVtdducs=
github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8=
github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vulcanize/vulcanizedb v0.0.5/go.mod h1:utXkheCL9VjTfmuivuvRiAAyHh54GSN9XRQNEbFCA8k=
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs=
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package integration
import (
@@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package integration
import (
@@ -17,14 +17,11 @@
package event
import (
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type Repository interface {
Create(headerID int64, models []interface{}) error
MarkHeaderChecked(headerID int64) error
MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
RecheckHeaders(startingBlockNumber, endingBlockNUmber int64) ([]core.Header, error)
SetDB(db *postgres.DB)
}
@@ -20,7 +20,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -37,7 +36,7 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.Event
return transformer
}
func (transformer Transformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
func (transformer Transformer) Execute(logs []types.Log, header core.Header) error {
transformerName := transformer.Config.TransformerName
config := transformer.Config
@@ -23,7 +23,6 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/libraries/shared/factories/event"
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
@@ -60,14 +59,14 @@ var _ = Describe("Transformer", func() {
})
It("marks header checked if no logs returned", func() {
err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
err := t.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedCalledWith(headerOne.Id)
})
It("doesn't attempt to convert or persist an empty collection when there are no logs", func() {
err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
err := t.Execute([]types.Log{}, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.ToEntitiesCalledCounter).To(Equal(0))
@@ -76,7 +75,7 @@ var _ = Describe("Transformer", func() {
})
It("does not call repository.MarkCheckedHeader when there are logs", func() {
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
repository.AssertMarkHeaderCheckedNotCalled()
@@ -85,14 +84,14 @@ var _ = Describe("Transformer", func() {
It("returns error if marking header checked returns err", func() {
repository.SetMarkHeaderCheckedError(fakes.FakeError)
err := t.Execute([]types.Log{}, headerOne, constants.HeaderMissing)
err := t.Execute([]types.Log{}, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
It("converts an eth log to an entity", func() {
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.ContractAbi).To(Equal(config.ContractAbi))
@@ -102,7 +101,7 @@ var _ = Describe("Transformer", func() {
It("returns an error if converter fails", func() {
converter.ToEntitiesError = fakes.FakeError
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -111,7 +110,7 @@ var _ = Describe("Transformer", func() {
It("converts an entity to a model", func() {
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(converter.EntitiesToConvert[0]).To(Equal(test_data.GenericEntity{}))
@@ -121,7 +120,7 @@ var _ = Describe("Transformer", func() {
converter.EntitiesToReturn = []interface{}{test_data.GenericEntity{}}
converter.ToModelsError = fakes.FakeError
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -130,7 +129,7 @@ var _ = Describe("Transformer", func() {
It("persists the record", func() {
converter.ModelsToReturn = []interface{}{test_data.GenericModel{}}
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedHeaderID).To(Equal(headerOne.Id))
@@ -139,7 +138,7 @@ var _ = Describe("Transformer", func() {
It("returns error if persisting the record fails", func() {
repository.SetCreateError(fakes.FakeError)
err := t.Execute(logs, headerOne, constants.HeaderMissing)
err := t.Execute(logs, headerOne)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
@@ -100,6 +100,7 @@ type Repository interface {
A contract-specific implementation of the repository interface enables the transformer to write the decoded storage value to the appropriate table in postgres.
The `Create` function is expected to recognize and persist a given storage value by the variable's name, as indicated on the row's metadata.
Note: we advise silently discarding duplicates in `Create` - as it's possible that you may read the same diff several times, and an error will trigger the storage watcher to queue that diff for later processing.
The `SetDB` function is required for the repository to connect to the database.
@@ -100,4 +100,54 @@ var _ = Describe("Storage transformer", func() {
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
Describe("when a storage row contains more than one item packed in storage", func() {
var (
rawValue = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
fakeBlockNumber = 123
fakeBlockHash = "0x67890"
packedTypes = make(map[int]utils.ValueType)
)
packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48
var fakeMetadata = utils.StorageValueMetadata{
Name: "",
Keys: nil,
Type: utils.PackedSlot,
PackedTypes: packedTypes,
}
It("passes the decoded data items to the repository", func() {
mappings.Metadata = fakeMetadata
fakeRow := utils.StorageDiffRow{
Contract: common.Address{},
BlockHash: common.HexToHash(fakeBlockHash),
BlockHeight: fakeBlockNumber,
StorageKey: common.Hash{},
StorageValue: rawValue.Hash(),
}
err := t.Execute(fakeRow)
Expect(err).NotTo(HaveOccurred())
Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
expectedPassedValue := make(map[int]string)
expectedPassedValue[0] = "10800"
expectedPassedValue[1] = "172800"
Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
})
It("returns error if creating a row fails", func() {
mappings.Metadata = fakeMetadata
repository.CreateErr = fakes.FakeError
err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError))
})
})
})
+5 -1
View File
@@ -17,9 +17,12 @@
package fetcher
import (
"strings"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/pkg/fs"
"strings"
)
type IStorageFetcher interface {
@@ -39,6 +42,7 @@ func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.S
if tailErr != nil {
errs <- tailErr
}
log.Debug("fetching storage diffs...")
for line := range t.Lines {
row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
if parseErr != nil {
+1 -2
View File
@@ -19,7 +19,6 @@ package mocks
import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
shared_t "github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -33,7 +32,7 @@ type MockTransformer struct {
config shared_t.EventTransformerConfig
}
func (mh *MockTransformer) Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error {
func (mh *MockTransformer) Execute(logs []types.Log, header core.Header) error {
if mh.ExecuteError != nil {
return mh.ExecuteError
}
+29 -43
View File
@@ -69,31 +69,6 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
return result, err
}
func RecheckHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.DB, checkedHeadersColumn string) ([]core.Header, error) {
var result []core.Header
var query string
var err error
if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2`
err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
} else {
query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE ` + checkedHeadersColumn + ` between 1 and ` + constants.RecheckHeaderCap + `
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`
err = db.Select(&result, query, startingBlockNumber, endingBlockNumber, db.Node.ID)
}
return result, err
}
func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
// Query returns `[]driver.Value`, nullable polymorphic interface
var queryResult []driver.Value
@@ -101,7 +76,8 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
`SELECT column_name FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'checked_headers'
AND column_name ~ '_checked';`
AND column_name <> 'id'
AND column_name <> 'header_id';`
err := db.Select(&queryResult, columnNamesQuery)
if err != nil {
@@ -121,37 +97,47 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
return columnNames, nil
}
// Builds a SQL string that checks if any column value is 0, given the column names.
// Builds a SQL string that checks if any column should be checked/rechecked.
// Defaults to FALSE when no columns are provided.
// Ex: ["columnA", "columnB"] => "NOT (columnA!=0 AND columnB!=0)"
// [] => "FALSE"
func CreateNotCheckedSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
var result bytes.Buffer
func CreateHeaderCheckedPredicateSQL(boolColumns []string, recheckHeaders constants.TransformerExecution) string {
if len(boolColumns) == 0 {
return "FALSE"
}
result.WriteString("NOT (")
if recheckHeaders {
return createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns)
} else {
return createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns)
}
}
func createHeaderCheckedPredicateSQLForMissingHeaders(boolColumns []string) string {
var result bytes.Buffer
result.WriteString(" (")
// Loop excluding last column name
for _, column := range boolColumns[:len(boolColumns)-1] {
if recheckHeaders {
result.WriteString(fmt.Sprintf("%v>=%s AND ", column, constants.RecheckHeaderCap))
} else {
result.WriteString(fmt.Sprintf("%v!=0 AND ", column))
}
result.WriteString(fmt.Sprintf("%v=0 OR ", column))
}
// No trailing "OR" for the last column name
if recheckHeaders {
result.WriteString(fmt.Sprintf("%v>=%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
} else {
result.WriteString(fmt.Sprintf("%v!=0)", boolColumns[len(boolColumns)-1]))
result.WriteString(fmt.Sprintf("%v=0)", boolColumns[len(boolColumns)-1]))
return result.String()
}
func createHeaderCheckedPredicateSQLForRecheckedHeaders(boolColumns []string) string {
var result bytes.Buffer
result.WriteString(" (")
// Loop excluding last column name
for _, column := range boolColumns[:len(boolColumns)-1] {
result.WriteString(fmt.Sprintf("%v<%s OR ", column, constants.RecheckHeaderCap))
}
// No trailing "OR" for the last column name
result.WriteString(fmt.Sprintf("%v<%s)", boolColumns[len(boolColumns)-1], constants.RecheckHeaderCap))
return result.String()
}
+85 -107
View File
@@ -36,12 +36,12 @@ import (
)
var _ = Describe("Repository", func() {
Describe("MarkHeaderChecked", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
var (
checkedHeadersColumn string
db *postgres.DB
)
Describe("MarkHeaderChecked", func() {
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
@@ -73,11 +73,6 @@ var _ = Describe("Repository", func() {
})
Describe("MarkHeaderCheckedInTransaction", func() {
var (
checkedHeadersColumn string
db *postgres.DB
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
@@ -114,16 +109,17 @@ var _ = Describe("Repository", func() {
Describe("MissingHeaders", func() {
var (
db *postgres.DB
headerRepository datastore.HeaderRepository
startingBlockNumber int64
endingBlockNumber int64
eventSpecificBlockNumber int64
outOfRangeBlockNumber int64
blockNumbers []int64
headerIDs []int64
notCheckedSQL string
err error
hr r2.HeaderRepository
columnNames []string
)
BeforeEach(func() {
@@ -133,14 +129,14 @@ var _ = Describe("Repository", func() {
hr = r2.NewHeaderRepository(db)
hr.AddCheckColumns(getExpectedColumnNames())
columnNames, err := shared.GetCheckedColumnNames(db)
columnNames, err = shared.GetCheckedColumnNames(db)
Expect(err).NotTo(HaveOccurred())
notCheckedSQL = shared.CreateNotCheckedSQL(columnNames, constants.HeaderMissing)
notCheckedSQL = shared.CreateHeaderCheckedPredicateSQL(columnNames, constants.HeaderMissing)
startingBlockNumber = rand.Int63()
eventSpecificBlockNumber = startingBlockNumber + 1
endingBlockNumber = startingBlockNumber + 2
outOfRangeBlockNumber := endingBlockNumber + 1
outOfRangeBlockNumber = endingBlockNumber + 1
blockNumbers = []int64{startingBlockNumber, eventSpecificBlockNumber, endingBlockNumber, outOfRangeBlockNumber}
@@ -157,6 +153,7 @@ var _ = Describe("Repository", func() {
})
It("only treats headers as checked if the event specific logs have been checked", func() {
//add a checked_header record, but don't mark it check for any of the columns
_, err = db.Exec(`INSERT INTO public.checked_headers (header_id) VALUES ($1)`, headerIDs[1])
Expect(err).NotTo(HaveOccurred())
@@ -191,83 +188,47 @@ var _ = Describe("Repository", func() {
Expect(nodeTwoMissingHeaders[0].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
Expect(nodeTwoMissingHeaders[1].BlockNumber).To(Or(Equal(startingBlockNumber+10), Equal(eventSpecificBlockNumber+10), Equal(endingBlockNumber+10)))
})
})
Describe("RecheckHeaders", func() {
var (
checkedHeadersColumn string
db *postgres.DB
headerOneID, headerTwoID, headerThreeID, headerFourID int64
headerOneErr, headerTwoErr, headerThreeErr, headerFourErr error
)
It("handles an ending block of -1 ", func() {
endingBlock := int64(-1)
headers, err := shared.MissingHeaders(startingBlockNumber, endingBlock, db, notCheckedSQL)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(4))
Expect(headers[0].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[1].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[2].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
Expect(headers[3].BlockNumber).To(Or(Equal(startingBlockNumber), Equal(endingBlockNumber), Equal(eventSpecificBlockNumber), Equal(outOfRangeBlockNumber)))
// create header checked column
checkedHeadersColumn = "test_column_checked"
_, migrateErr := db.Exec(`ALTER TABLE public.checked_headers ADD COLUMN ` + checkedHeadersColumn + ` integer`)
Expect(migrateErr).NotTo(HaveOccurred())
})
// create headers
headerRepository := repositories.NewHeaderRepository(db)
headerOneID, headerOneErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(1))
Expect(headerOneErr).NotTo(HaveOccurred())
headerTwoID, headerTwoErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(2))
Expect(headerTwoErr).NotTo(HaveOccurred())
headerThreeID, headerThreeErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(3))
Expect(headerThreeErr).NotTo(HaveOccurred())
headerFourID, headerFourErr = headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(4))
Expect(headerFourErr).NotTo(HaveOccurred())
It("when a the `notCheckedSQL` argument allows for rechecks it returns headers where the checked count is less than the maximum", func() {
columnName := columnNames[0]
recheckedSQL := shared.CreateHeaderCheckedPredicateSQL([]string{columnName}, constants.HeaderRecheck)
// mark every header checked at least once
// header 4 is marked the maximum number of times, it it is not longer checked
// mark every header checked at least once, with one fully rechecked (headerThree)
maxCheckCount, intConversionErr := strconv.Atoi(constants.RecheckHeaderCap)
Expect(intConversionErr).NotTo(HaveOccurred())
_, markHeaderOneCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerOneID, maxCheckCount)
Expect(markHeaderOneCheckedErr).NotTo(HaveOccurred())
_, markHeaderTwoCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerTwoID, maxCheckCount)
Expect(markHeaderTwoCheckedErr).NotTo(HaveOccurred())
_, markHeaderThreeCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerThreeID, maxCheckCount+1)
Expect(markHeaderThreeCheckedErr).NotTo(HaveOccurred())
_, markHeaderFourCheckedErr := db.Exec(
`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) VALUES ($1, $2)`,
headerFourID, maxCheckCount)
Expect(markHeaderFourCheckedErr).NotTo(HaveOccurred())
})
AfterEach(func() {
_, cleanupMigrateErr := db.Exec(`ALTER TABLE public.checked_headers DROP COLUMN ` + checkedHeadersColumn)
Expect(cleanupMigrateErr).NotTo(HaveOccurred())
})
markHeaderOneErr := shared.MarkHeaderChecked(headerIDs[0], db, columnName)
Expect(markHeaderOneErr).NotTo(HaveOccurred())
markHeaderTwoErr := shared.MarkHeaderChecked(headerIDs[1], db, columnName)
Expect(markHeaderTwoErr).NotTo(HaveOccurred())
markHeaderThreeErr := shared.MarkHeaderChecked(headerIDs[2], db, columnName)
Expect(markHeaderThreeErr).NotTo(HaveOccurred())
for i := 0; i <= maxCheckCount; i++ {
markHeaderFourErr := shared.MarkHeaderChecked(headerIDs[3], db, columnName)
Expect(markHeaderFourErr).NotTo(HaveOccurred())
}
Describe("when no ending block number (ending block number == -1)", func() {
It("returns all headers since starting block where checked count is less than cap", func() {
headers, err := shared.RecheckHeaders(1, -1, db, checkedHeadersColumn)
headers, err := shared.MissingHeaders(1, -1, db, recheckedSQL)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
Expect(headers[2].Id).To(Or(Equal(headerOneID), Equal(headerTwoID), Equal(headerFourID)))
})
})
Describe("when ending block number specified", func() {
It("returns headers between starting and ending block where checked count is less than cap", func() {
headers, err := shared.RecheckHeaders(1, 3, db, checkedHeadersColumn)
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(2))
Expect(headers[0].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
Expect(headers[1].Id).To(Or(Equal(headerOneID), Equal(headerTwoID)))
})
Expect(err).NotTo(HaveOccurred())
Expect(len(headers)).To(Equal(3))
Expect(headers[0].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
Expect(headers[1].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
Expect(headers[2].Id).To(Or(Equal(headerIDs[0]), Equal(headerIDs[1]), Equal(headerIDs[2])))
})
})
@@ -285,41 +246,58 @@ var _ = Describe("Repository", func() {
})
})
Describe("CreateNotCheckedSQL", func() {
It("generates a correct SQL string for one column", func() {
columns := []string{"columnA"}
expected := "NOT (columnA!=0)"
actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
Describe("CreateHeaderCheckedPredicateSQL", func() {
Describe("for headers that haven't been checked for logs", func() {
It("generates a correct SQL string for one column", func() {
columns := []string{"columnA"}
expected := " (columnA=0)"
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
})
It("generates a correct SQL string for several columns", func() {
columns := []string{"columnA", "columnB"}
expected := " (columnA=0 OR columnB=0)"
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
})
It("defaults to FALSE when there are no columns", func() {
expected := "FALSE"
actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
})
})
It("generates a correct SQL string for several columns", func() {
columns := []string{"columnA", "columnB"}
expected := "NOT (columnA!=0 AND columnB!=0)"
actual := shared.CreateNotCheckedSQL(columns, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
})
Describe("for headers that are being rechecked for logs", func() {
It("generates a correct SQL string for rechecking headers for one column", func() {
columns := []string{"columnA"}
expected := fmt.Sprintf(" (columnA<%s)", constants.RecheckHeaderCap)
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
Expect(actual).To(Equal(expected))
})
It("defaults to FALSE when there are no columns", func() {
expected := "FALSE"
actual := shared.CreateNotCheckedSQL([]string{}, constants.HeaderMissing)
Expect(actual).To(Equal(expected))
})
It("generates a correct SQL string for rechecking headers for several columns", func() {
columns := []string{"columnA", "columnB"}
expected := fmt.Sprintf(" (columnA<%s OR columnB<%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
actual := shared.CreateHeaderCheckedPredicateSQL(columns, constants.HeaderRecheck)
Expect(actual).To(Equal(expected))
})
It("generates a correct SQL string for rechecking headers", func() {
columns := []string{"columnA", "columnB"}
expected := fmt.Sprintf("NOT (columnA>=%s AND columnB>=%s)", constants.RecheckHeaderCap, constants.RecheckHeaderCap)
actual := shared.CreateNotCheckedSQL(columns, constants.HeaderRecheck)
Expect(actual).To(Equal(expected))
It("defaults to FALSE when there are no columns", func() {
expected := "FALSE"
actual := shared.CreateHeaderCheckedPredicateSQL([]string{}, constants.HeaderRecheck)
Expect(actual).To(Equal(expected))
})
})
})
})
func getExpectedColumnNames() []string {
return []string{
"column_1_checked",
"column_2_checked",
"column_3_checked",
"column_4_checked",
"column_1",
"column_2",
"column_3",
"column_4",
}
}
+1 -1
View File
@@ -47,7 +47,7 @@ const (
)
func GetMapping(indexOnContract, key string) common.Hash {
keyBytes := common.FromHex("0x" + key + indexOnContract)
keyBytes := common.FromHex(key + indexOnContract)
encoded := crypto.Keccak256(keyBytes)
return common.BytesToHash(encoded)
}
+10
View File
@@ -21,6 +21,16 @@ var _ = Describe("Mappings", func() {
expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
Expect(storageKey).To(Equal(expectedStorageKey))
})
It("returns same result if value includes hex prefix", func() {
indexOfMappingOnContract := storage.IndexZero
keyForDesiredValueInMapping := "0x1234567890abcdef"
storageKey := storage.GetMapping(indexOfMappingOnContract, keyForDesiredValueInMapping)
expectedStorageKey := common.HexToHash("0xee0c1b59a3856bafbfb8730e7694c4badc271eb5f01ce4a8d7a53d8a6499676f")
Expect(storageKey).To(Equal(expectedStorageKey))
})
})
Describe("GetNestedMapping", func() {
+1 -1
View File
@@ -38,7 +38,7 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
func (queue StorageQueue) Add(row utils.StorageDiffRow) error {
_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
block_hash, block_height, storage_key, storage_value) VALUES
($1, $2, $3, $4, $5)`, row.Contract.Bytes(), row.BlockHash.Bytes(),
($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, row.Contract.Bytes(), row.BlockHash.Bytes(),
row.BlockHeight, row.StorageKey.Bytes(), row.StorageValue.Bytes())
return err
}
+16 -5
View File
@@ -49,11 +49,22 @@ var _ = Describe("Storage queue", func() {
Expect(addErr).NotTo(HaveOccurred())
})
It("adds a storage row to the db", func() {
var result utils.StorageDiffRow
getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
Expect(getErr).NotTo(HaveOccurred())
Expect(result).To(Equal(row))
Describe("Add", func() {
It("adds a storage row to the db", func() {
var result utils.StorageDiffRow
getErr := db.Get(&result, `SELECT contract, block_hash, block_height, storage_key, storage_value FROM public.queued_storage`)
Expect(getErr).NotTo(HaveOccurred())
Expect(result).To(Equal(row))
})
It("does not duplicate storage rows", func() {
addErr := queue.Add(row)
Expect(addErr).NotTo(HaveOccurred())
var count int
getErr := db.Get(&count, `SELECT count(*) FROM public.queued_storage`)
Expect(getErr).NotTo(HaveOccurred())
Expect(count).To(Equal(1))
})
})
It("deletes storage row from db", func() {
+61 -8
View File
@@ -23,27 +23,30 @@ import (
"github.com/ethereum/go-ethereum/common"
)
const (
bitsPerByte = 8
)
func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
switch metadata.Type {
case Uint256:
return decodeUint256(row.StorageValue.Bytes()), nil
return decodeInteger(row.StorageValue.Bytes()), nil
case Uint48:
return decodeUint48(row.StorageValue.Bytes()), nil
return decodeInteger(row.StorageValue.Bytes()), nil
case Uint128:
return decodeInteger(row.StorageValue.Bytes()), nil
case Address:
return decodeAddress(row.StorageValue.Bytes()), nil
case Bytes32:
return row.StorageValue.Hex(), nil
case PackedSlot:
return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
default:
panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
}
}
func decodeUint256(raw []byte) string {
n := big.NewInt(0).SetBytes(raw)
return n.String()
}
func decodeUint48(raw []byte) string {
func decodeInteger(raw []byte) string {
n := big.NewInt(0).SetBytes(raw)
return n.String()
}
@@ -51,3 +54,53 @@ func decodeUint48(raw []byte) string {
func decodeAddress(raw []byte) string {
return common.BytesToAddress(raw).Hex()
}
func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string {
storageSlotData := raw
decodedStorageSlotItems := map[int]string{}
numberOfTypes := len(packedTypes)
for position := 0; position < numberOfTypes; position++ {
//get length of remaining storage date
lengthOfStorageData := len(storageSlotData)
//get item details (type, length, starting index, value bytes)
itemType := packedTypes[position]
lengthOfItem := getNumberOfBytes(itemType)
itemStartingIndex := lengthOfStorageData - lengthOfItem
itemValueBytes := storageSlotData[itemStartingIndex:]
//decode item's bytes and set in results map
decodedValue := decodeIndividualItem(itemValueBytes, itemType)
decodedStorageSlotItems[position] = decodedValue
//pop last item off raw slot data before moving on
storageSlotData = storageSlotData[0:itemStartingIndex]
}
return decodedStorageSlotItems
}
func decodeIndividualItem(itemBytes []byte, valueType ValueType) string {
switch valueType {
case Uint48, Uint128:
return decodeInteger(itemBytes)
case Address:
return decodeAddress(itemBytes)
default:
panic(fmt.Sprintf("can't decode unknown type: %d", valueType))
}
}
func getNumberOfBytes(valueType ValueType) int {
switch valueType {
case Uint48:
return 48 / bitsPerByte
case Uint128:
return 128 / bitsPerByte
case Address:
return 20
default:
panic(fmt.Sprintf("ValueType %d not recognized", valueType))
}
}
@@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
})
It("decodes uint128", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
row := utils.StorageDiffRow{StorageValue: fakeInt}
metadata := utils.StorageValueMetadata{Type: utils.Uint128}
result, err := utils.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
})
It("decodes uint48", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
row := utils.StorageDiffRow{StorageValue: fakeInt}
@@ -59,4 +70,105 @@ var _ = Describe("Storage decoder", func() {
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(fakeAddress.Hex()))
})
Describe("when there are multiple items packed in the storage slot", func() {
It("decodes uint48 items", func() {
//this is a real storage data example
packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
row := utils.StorageDiffRow{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48
metadata := utils.StorageValueMetadata{
Type: utils.PackedSlot,
PackedTypes: packedTypes,
}
result, err := utils.Decode(row, metadata)
decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred())
Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
})
It("decodes 5 uint48 items", func() {
//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
//real storage data that has several items packed into it
packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
packedStorage := common.HexToHash(packedStorageHex)
row := utils.StorageDiffRow{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48
packedTypes[2] = utils.Uint48
packedTypes[3] = utils.Uint48
packedTypes[4] = utils.Uint48
metadata := utils.StorageValueMetadata{
Type: utils.PackedSlot,
PackedTypes: packedTypes,
}
result, err := utils.Decode(row, metadata)
decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred())
Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String()))
Expect(decodedValues[3]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String()))
Expect(decodedValues[4]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String()))
})
It("decodes 2 uint128 items", func() {
//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
//real storage data that has several items packed into it
packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
"0000000000000000AB54A98CEB1F0AD2"
packedStorage := common.HexToHash(packedStorageHex)
row := utils.StorageDiffRow{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint128
packedTypes[1] = utils.Uint128
metadata := utils.StorageValueMetadata{
Type: utils.PackedSlot,
PackedTypes: packedTypes,
}
result, err := utils.Decode(row, metadata)
decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred())
Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String()))
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("38D7EA4C67FF8E502B6730000").Bytes()).String()))
})
It("decodes address + 2 uint48s", func() {
//TODO: replace with real data when available
addressHex := "0000000000000000000000000000000000012345"
packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
row := utils.StorageDiffRow{StorageValue: packedStorage}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Address
packedTypes[1] = utils.Uint48
packedTypes[2] = utils.Uint48
metadata := utils.StorageValueMetadata{
Type: utils.PackedSlot,
PackedTypes: packedTypes,
}
result, err := utils.Decode(row, metadata)
decodedValues := result.(map[int]string)
Expect(err).NotTo(HaveOccurred())
Expect(decodedValues[0]).To(Equal("0x" + addressHex))
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
})
})
})
-3
View File
@@ -17,12 +17,9 @@
package utils
import (
"errors"
"fmt"
)
var ErrRowExists = errors.New("parsed row for storage diff already exists")
type ErrContractNotFound struct {
Contract string
}
+34 -7
View File
@@ -16,27 +16,54 @@
package utils
import "fmt"
type ValueType int
const (
Uint256 ValueType = iota
Uint48
Uint128
Bytes32
Address
PackedSlot
)
type Key string
type StorageValueMetadata struct {
Name string
Keys map[Key]string
Type ValueType
Name string
Keys map[Key]string
Type ValueType
PackedNames map[int]string //zero indexed position in map => name of packed item
PackedTypes map[int]ValueType //zero indexed position in map => type of packed item
}
func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata {
func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata {
return getMetadata(name, keys, valueType, nil, nil)
}
func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
return getMetadata(name, keys, valueType, packedNames, packedTypes)
}
func getMetadata(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
assertPackedSlotArgs(valueType, packedNames, packedTypes)
return StorageValueMetadata{
Name: name,
Keys: keys,
Type: t,
Name: name,
Keys: keys,
Type: valueType,
PackedNames: packedNames,
PackedTypes: packedTypes,
}
}
func assertPackedSlotArgs(valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) {
if valueType == PackedSlot && (packedTypes == nil || packedNames == nil) {
panic(fmt.Sprintf("ValueType is PackedSlot. Expected PackedNames and PackedTypes to not be nil, but got PackedNames = %v and PackedTypes = %v", packedNames, packedTypes))
} else if (packedNames != nil && packedTypes != nil) && valueType != PackedSlot {
panic(fmt.Sprintf("PackedNames and PackedTypes passed in. Expected ValueType to equal PackedSlot (%v), but got %v.", PackedSlot, valueType))
}
}
+57 -1
View File
@@ -7,7 +7,7 @@ import (
)
var _ = Describe("Storage value metadata getter", func() {
It("returns a storage value metadata instance with corresponding fields assigned", func() {
It("returns storage value metadata for a single storage variable", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.Uint256
@@ -19,4 +19,60 @@ var _ = Describe("Storage value metadata getter", func() {
}
Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
})
Describe("metadata for a packed storaged slot", func() {
It("returns metadata for multiple storage variables", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.PackedSlot
metadataPackedNames := map[int]string{0: "name"}
metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
expectedMetadata := utils.StorageValueMetadata{
Name: metadataName,
Keys: metadataKeys,
Type: metadataType,
PackedTypes: metadataPackedTypes,
PackedNames: metadataPackedNames,
}
Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata))
})
It("panics if PackedTypes are nil when the type is PackedSlot", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.PackedSlot
metadataPackedNames := map[int]string{0: "name"}
getMetadata := func() {
utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, nil)
}
Expect(getMetadata).To(Panic())
})
It("panics if PackedNames are nil when the type is PackedSlot", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.PackedSlot
metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
getMetadata := func() {
utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, nil, metadataPackedTypes)
}
Expect(getMetadata).To(Panic())
})
It("panics if valueType is not PackedSlot if PackedNames is populated", func() {
metadataName := "fake_name"
metadataKeys := map[utils.Key]string{"key": "value"}
metadataType := utils.Uint48
metadataPackedNames := map[int]string{0: "name"}
metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
getMetadata := func() {
utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)
}
Expect(getMetadata).To(Panic())
})
})
})
@@ -20,13 +20,12 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/libraries/shared/constants"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type EventTransformer interface {
Execute(logs []types.Log, header core.Header, recheckHeaders constants.TransformerExecution) error
Execute(logs []types.Log, header core.Header) error
GetConfig() EventTransformerConfig
}
+9 -4
View File
@@ -96,11 +96,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
if err != nil {
return err
}
notCheckedSQL := repository.CreateNotCheckedSQL(checkedColumnNames, recheckHeaders)
notCheckedSQL := repository.CreateHeaderCheckedPredicateSQL(checkedColumnNames, recheckHeaders)
missingHeaders, err := repository.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
if err != nil {
logrus.Error("Fetching of missing headers failed in watcher!")
logrus.Error("Couldn't fetch missing headers in watcher: ", err)
return err
}
@@ -108,7 +108,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
// TODO Extend FetchLogs for doing several blocks at a time
logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
if err != nil {
logrus.Errorf("Error while fetching logs for header %v in watcher", header.Id)
logrus.WithFields(logrus.Fields{
"headerId": header.Id,
"headerHash": header.Hash,
"blockNumber": header.BlockNumber,
}).Errorf("Couldn't fetch logs for header: %v", err)
return err
}
@@ -120,6 +124,7 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
transformErr := watcher.transformLogs(logs, header)
if transformErr != nil {
logrus.Error("Could not transform logs: ", transformErr)
return transformErr
}
}
@@ -134,7 +139,7 @@ func (watcher *EventWatcher) transformLogs(logs []types.Log, header core.Header)
for _, t := range watcher.Transformers {
transformerName := t.GetConfig().TransformerName
logChunk := chunkedLogs[transformerName]
err := t.Execute(logChunk, header, constants.HeaderMissing)
err := t.Execute(logChunk, header)
if err != nil {
logrus.Errorf("%v transformer failed to execute in watcher: %v", transformerName, err)
return err
+2 -2
View File
@@ -78,7 +78,7 @@ func (storageWatcher StorageWatcher) processRow(row utils.StorageDiffRow) {
return
}
executeErr := storageTransformer.Execute(row)
if executeErr != nil && executeErr != utils.ErrRowExists {
if executeErr != nil {
logrus.Warn(fmt.Sprintf("error executing storage transformer: %s", executeErr))
queueErr := storageWatcher.Queue.Add(row)
if queueErr != nil {
@@ -100,7 +100,7 @@ func (storageWatcher StorageWatcher) processQueue() {
continue
}
executeErr := storageTransformer.Execute(row)
if executeErr == nil || executeErr == utils.ErrRowExists {
if executeErr == nil {
storageWatcher.deleteRow(row.Id)
}
}
@@ -109,18 +109,6 @@ var _ = Describe("Storage Watcher", func() {
close(done)
})
It("does not queue row if transformer execution fails because row already exists", func(done Done) {
mockTransformer.ExecuteErr = utils.ErrRowExists
go storageWatcher.Execute(rows, errs, time.Hour)
Expect(<-errs).To(BeNil())
Consistently(func() bool {
return mockQueue.AddCalled
}).Should(BeFalse())
close(done)
})
It("queues row for later processing if transformer execution fails", func(done Done) {
mockTransformer.ExecuteErr = fakes.FakeError
@@ -199,17 +187,6 @@ var _ = Describe("Storage Watcher", func() {
close(done)
})
It("deletes row from queue if transformer execution errors because row already exists", func(done Done) {
mockTransformer.ExecuteErr = utils.ErrRowExists
go storageWatcher.Execute(rows, errs, time.Nanosecond)
Eventually(func() int {
return mockQueue.DeletePassedId
}).Should(Equal(row.Id))
close(done)
})
It("logs error if deleting persisted row fails", func(done Done) {
mockQueue.DeleteErr = fakes.FakeError
tempFile, fileErr := ioutil.TempFile("", "log")
+5 -2
View File
@@ -3,12 +3,15 @@ package main
import (
"github.com/vulcanize/vulcanizedb/cmd"
log "github.com/sirupsen/logrus"
"os"
log "github.com/sirupsen/logrus"
)
func main() {
log.SetFormatter(&log.JSONFormatter{})
log.SetFormatter(&log.JSONFormatter{
PrettyPrint: true,
})
file, err := os.OpenFile("vulcanizedb.log",
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err == nil {
@@ -17,7 +17,9 @@
package retriever
import (
"database/sql"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
// Block retriever is used to retrieve the first block for a given contract and the most recent block
@@ -41,7 +43,10 @@ func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
if err != nil {
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
if err == sql.ErrNoRows {
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
}
return i, err
}
return i, err
@@ -49,18 +54,26 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
var firstBlock int
var firstBlock int64
addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr)
if getAddressErr != nil {
return firstBlock, getAddressErr
}
err := r.db.Get(
&firstBlock,
`SELECT number FROM blocks
WHERE id = (SELECT block_id FROM full_sync_receipts
WHERE lower(contract_address) = $1
WHERE contract_address_id = $1
ORDER BY block_id ASC
LIMIT 1)`,
contractAddr,
addressId,
)
return int64(firstBlock), err
return firstBlock, err
}
func addressRepository() repositories.AddressRepository {
return repositories.AddressRepository{}
}
// In which case this servers as a heuristic to find the first block by finding the first contract event log
@@ -164,7 +164,7 @@ func (tr *Transformer) Init() error {
// Iterates through stored, initialized contract objects
// Iterates through contract's event filters, grabbing watched event logs
// Uses converter to convert logs into custom log type
// Persists converted logs into custuom postgres tables
// Persists converted logs into custom postgres tables
// Calls selected methods, using token holder address generated during event log conversion
func (tr *Transformer) Execute() error {
if len(tr.Contracts) == 0 {
@@ -137,7 +137,7 @@ func SetupTusdRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string)
}, core.Node{})
Expect(err).NotTo(HaveOccurred())
receiptRepository := repositories.ReceiptRepository{DB: db}
receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
logRepository := repositories.LogRepository{DB: db}
blockRepository := *repositories.NewBlockRepository(db)
@@ -183,7 +183,7 @@ func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (
}, core.Node{})
Expect(err).NotTo(HaveOccurred())
receiptRepository := repositories.ReceiptRepository{DB: db}
receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
logRepository := repositories.LogRepository{DB: db}
blockRepository := *repositories.NewBlockRepository(db)
@@ -225,6 +225,9 @@ func TearDown(db *postgres.DB) {
tx, err := db.Beginx()
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM addresses`)
Expect(err).NotTo(HaveOccurred())
_, err = tx.Exec(`DELETE FROM blocks`)
Expect(err).NotTo(HaveOccurred())
@@ -220,7 +220,7 @@ var _ = Describe("Parser", func() {
Expect(err).ToNot(HaveOccurred())
selectMethods := p.GetMethods([]string{})
Expect(len(selectMethods)).To(Equal(22))
Expect(len(selectMethods)).To(Equal(25))
})
})
})
+1 -1
View File
@@ -29,7 +29,7 @@ type Block struct {
Number int64 `db:"number"`
ParentHash string `db:"parent_hash"`
Size string `db:"size"`
Time int64 `db:"time"`
Time uint64 `db:"time"`
Transactions []TransactionModel
UncleHash string `db:"uncle_hash"`
UnclesReward string `db:"uncles_reward"`
+1 -1
View File
@@ -30,7 +30,7 @@ type BlockChain interface {
GetBlockByNumber(blockNumber int64) (Block, error)
GetEthLogsWithCustomQuery(query ethereum.FilterQuery) ([]types.Log, error)
GetHeaderByNumber(blockNumber int64) (Header, error)
GetHeaderByNumbers(blockNumbers []int64) ([]Header, error)
GetHeadersByNumbers(blockNumbers []int64) ([]Header, error)
GetLogs(contract Contract, startingBlockNumber *big.Int, endingBlockNumber *big.Int) ([]Log, error)
GetTransactions(transactionHashes []common.Hash) ([]TransactionModel, error)
LastBlock() (*big.Int, error)
+1 -1
View File
@@ -42,7 +42,7 @@ type POAHeader struct {
Number *hexutil.Big `json:"number" gencodec:"required"`
GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"`
GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
Time hexutil.Uint64 `json:"timestamp" gencodec:"required"`
Extra hexutil.Bytes `json:"extraData" gencodec:"required"`
Hash common.Hash `json:"hash"`
}
@@ -21,6 +21,7 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
)
type Reader interface {
@@ -48,7 +49,7 @@ func (ldbr *LevelDatabaseReader) GetBlockNumber(hash common.Hash) *uint64 {
}
func (ldbr *LevelDatabaseReader) GetBlockReceipts(hash common.Hash, number uint64) types.Receipts {
return rawdb.ReadReceipts(ldbr.reader, hash, number)
return rawdb.ReadReceipts(ldbr.reader, hash, number, &params.ChainConfig{})
}
func (ldbr *LevelDatabaseReader) GetCanonicalHash(number uint64) common.Hash {
@@ -0,0 +1,62 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories
import (
"database/sql"
"github.com/ethereum/go-ethereum/common"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type AddressRepository struct{}
func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
stringAddressToCommonAddress := common.HexToAddress(address)
hexAddress := stringAddressToCommonAddress.Hex()
var addressId int64
getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
if getErr == sql.ErrNoRows {
insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
return addressId, insertErr
}
return addressId, getErr
}
func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
stringAddressToCommonAddress := common.HexToAddress(address)
hexAddress := stringAddressToCommonAddress.Hex()
var addressId int64
getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
if getErr == sql.ErrNoRows {
insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
return addressId, insertErr
}
return addressId, getErr
}
func (AddressRepository) GetAddressById(db *postgres.DB, id int64) (string, error) {
var address string
getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
return address, getErr
}
@@ -0,0 +1,181 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories_test
import (
"strings"
"github.com/jmoiron/sqlx"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("address lookup", func() {
var (
db *postgres.DB
repo repositories.AddressRepository
address = fakes.FakeAddress.Hex()
)
BeforeEach(func() {
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
repo = repositories.AddressRepository{}
})
AfterEach(func() {
test_config.CleanTestDB(db)
})
type dbAddress struct {
Id int64
Address string
}
Describe("GetOrCreateAddress", func() {
It("creates an address record", func() {
addressId, createErr := repo.GetOrCreateAddress(db, address)
Expect(createErr).NotTo(HaveOccurred())
var actualAddress dbAddress
getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)
Expect(getErr).NotTo(HaveOccurred())
expectedAddress := dbAddress{Id: addressId, Address: address}
Expect(actualAddress).To(Equal(expectedAddress))
})
It("returns the existing record id if the address already exists", func() {
createId, createErr := repo.GetOrCreateAddress(db, address)
Expect(createErr).NotTo(HaveOccurred())
getId, getErr := repo.GetOrCreateAddress(db, address)
Expect(getErr).NotTo(HaveOccurred())
var addressCount int
addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`)
Expect(addressErr).NotTo(HaveOccurred())
Expect(addressCount).To(Equal(1))
Expect(createId).To(Equal(getId))
})
It("gets upper-cased addresses", func() {
upperAddress := strings.ToUpper(address)
upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
Expect(getErr).NotTo(HaveOccurred())
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
})
It("gets lower-cased addresses", func() {
lowerAddress := strings.ToLower(address)
upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
Expect(getErr).NotTo(HaveOccurred())
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
})
})
Describe("GetOrCreateAddressInTransaction", func() {
var (
tx *sqlx.Tx
txErr error
)
BeforeEach(func() {
tx, txErr = db.Beginx()
Expect(txErr).NotTo(HaveOccurred())
})
AfterEach(func() {
tx.Rollback()
})
It("creates an address record", func() {
addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
Expect(createErr).NotTo(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).NotTo(HaveOccurred())
var actualAddress dbAddress
getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)
Expect(getErr).NotTo(HaveOccurred())
expectedAddress := dbAddress{Id: addressId, Address: address}
Expect(actualAddress).To(Equal(expectedAddress))
})
It("returns the existing record id if the address already exists", func() {
_, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
Expect(createErr).NotTo(HaveOccurred())
_, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
Expect(getErr).NotTo(HaveOccurred())
tx.Commit()
var addressCount int
addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`)
Expect(addressErr).NotTo(HaveOccurred())
})
It("gets upper-cased addresses", func() {
upperAddress := strings.ToUpper(address)
upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
Expect(getErr).NotTo(HaveOccurred())
tx.Commit()
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
})
It("gets lower-cased addresses", func() {
lowerAddress := strings.ToLower(address)
upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress)
Expect(createErr).NotTo(HaveOccurred())
mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
Expect(getErr).NotTo(HaveOccurred())
tx.Commit()
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
})
})
Describe("GetAddressById", func() {
It("gets and address by it's id", func() {
addressId, createErr := repo.GetOrCreateAddress(db, address)
Expect(createErr).NotTo(HaveOccurred())
actualAddress, getErr := repo.GetAddressById(db, addressId)
Expect(getErr).NotTo(HaveOccurred())
Expect(actualAddress).To(Equal(address))
})
It("returns an error if the id doesn't exist", func() {
_, getErr := repo.GetAddressById(db, 0)
Expect(getErr).To(HaveOccurred())
Expect(getErr).To(MatchError("sql: no rows in result set"))
})
})
})
@@ -234,7 +234,8 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in
return err
}
if hasReceipt(transaction) {
receiptId, err := blockRepository.createReceipt(tx, blockId, transaction.Receipt)
receiptRepo := FullSyncReceiptRepository{}
receiptId, err := receiptRepo.CreateFullSyncReceiptInTx(blockId, transaction.Receipt, tx)
if err != nil {
return err
}
@@ -256,22 +257,6 @@ func hasReceipt(transaction core.TransactionModel) bool {
return transaction.Receipt.TxHash != ""
}
func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) {
//Not currently persisting log bloom filters
var receiptId int
err := tx.QueryRow(
`INSERT INTO full_sync_receipts
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id`,
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId)
if err != nil {
log.Error("createReceipt: error inserting receipt: ", err)
return receiptId, err
}
return receiptId, nil
}
func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, bool) {
var retrievedBlockHash string
// TODO: handle possible error
@@ -283,7 +268,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
return retrievedBlockHash, blockExists(retrievedBlockHash)
}
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error {
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int64) error {
for _, tlog := range logs {
_, err := tx.Exec(
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
@@ -80,7 +80,7 @@ var _ = Describe("Saving blocks", func() {
blockNonce := "0x881db2ca900682e9a9"
miner := "x123"
extraData := "xextraData"
blockTime := int64(1508981640)
blockTime := uint64(1508981640)
uncleHash := "x789"
blockSize := string("1000")
difficulty := int64(10)
@@ -98,7 +98,7 @@ var _ = Describe("Saving blocks", func() {
Number: blockNumber,
ParentHash: blockParentHash,
Size: blockSize,
Time: blockTime,
Time: uint64(blockTime),
UncleHash: uncleHash,
UnclesReward: unclesReward,
}
@@ -26,17 +26,17 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
)
type ReceiptRepository struct {
type FullSyncReceiptRepository struct {
*postgres.DB
}
func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
func (receiptRepository FullSyncReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
tx, err := receiptRepository.DB.Beginx()
if err != nil {
return err
}
for _, receipt := range receipts {
receiptId, err := createReceipt(receipt, blockId, tx)
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, receipt, tx)
if err != nil {
tx.Rollback()
return err
@@ -53,21 +53,6 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64,
return nil
}
func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
var receiptId int64
err := tx.QueryRow(
`INSERT INTO full_sync_receipts
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id`,
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId,
).Scan(&receiptId)
if err != nil {
logrus.Error("createReceipt: Error inserting: ", err)
}
return receiptId, err
}
func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
for _, log := range logs {
_, err := tx.Exec(
@@ -83,27 +68,30 @@ func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
return nil
}
func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
tx, _ := receiptRepository.DB.Beginx()
func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
var receiptId int64
addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
if getAddressErr != nil {
logrus.Error("createReceipt: Error getting address id: ", getAddressErr)
return receiptId, getAddressErr
}
err := tx.QueryRow(
`INSERT INTO full_sync_receipts
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
(contract_address_id, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id`,
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId)
addressId, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId)
if err != nil {
tx.Rollback()
logrus.Warning("CreateReceipt: error inserting receipt: ", err)
return receiptId, err
}
tx.Commit()
return receiptId, nil
}
func (receiptRepository ReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) {
func (receiptRepository FullSyncReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) {
row := receiptRepository.DB.QueryRow(
`SELECT contract_address,
`SELECT contract_address_id,
tx_hash,
cumulative_gas_used,
gas_used,
@@ -30,7 +30,7 @@ import (
var _ = Describe("Receipt Repository", func() {
var blockRepository datastore.BlockRepository
var logRepository datastore.LogRepository
var receiptRepository datastore.ReceiptRepository
var receiptRepository datastore.FullSyncReceiptRepository
var db *postgres.DB
var node core.Node
BeforeEach(func() {
@@ -44,7 +44,7 @@ var _ = Describe("Receipt Repository", func() {
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
logRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
})
Describe("Saving multiple receipts", func() {
@@ -84,12 +84,12 @@ var _ = Describe("Receipt Repository", func() {
Expect(err).NotTo(HaveOccurred())
persistedReceiptOne, err := receiptRepository.GetReceipt(txHashOne)
persistedReceiptOne, err := receiptRepository.GetFullSyncReceipt(txHashOne)
Expect(err).NotTo(HaveOccurred())
Expect(persistedReceiptOne).NotTo(BeNil())
Expect(persistedReceiptOne.TxHash).To(Equal(txHashOne))
persistedReceiptTwo, err := receiptRepository.GetReceipt(txHashTwo)
persistedReceiptTwo, err := receiptRepository.GetFullSyncReceipt(txHashTwo)
Expect(err).NotTo(HaveOccurred())
Expect(persistedReceiptTwo).NotTo(BeNil())
Expect(persistedReceiptTwo.TxHash).To(Equal(txHashTwo))
@@ -124,7 +124,7 @@ var _ = Describe("Receipt Repository", func() {
_, err := blockRepository.CreateOrUpdateBlock(block)
Expect(err).NotTo(HaveOccurred())
receipt, err := receiptRepository.GetReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
receipt, err := receiptRepository.GetFullSyncReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
Expect(err).ToNot(HaveOccurred())
//Not currently serializing bloom logs
Expect(receipt.Bloom).To(Equal(core.Receipt{}.Bloom))
@@ -136,7 +136,7 @@ var _ = Describe("Receipt Repository", func() {
})
It("returns ErrReceiptDoesNotExist when receipt does not exist", func() {
receipt, err := receiptRepository.GetReceipt("DOES NOT EXIST")
receipt, err := receiptRepository.GetFullSyncReceipt("DOES NOT EXIST")
Expect(err).To(HaveOccurred())
Expect(receipt).To(BeZero())
})
@@ -154,7 +154,7 @@ var _ = Describe("Receipt Repository", func() {
_, err := blockRepository.CreateOrUpdateBlock(block)
Expect(err).NotTo(HaveOccurred())
_, err = receiptRepository.GetReceipt(receipt.TxHash)
_, err = receiptRepository.GetFullSyncReceipt(receipt.TxHash)
Expect(err).To(Not(HaveOccurred()))
})
})
@@ -85,22 +85,6 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i
return txId, err
}
func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) {
var receiptId int64
err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
(header_id, transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, transaction_id) DO UPDATE
SET (contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9)
RETURNING id`,
headerID, transactionID, receipt.ContractAddress, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId)
if err != nil {
log.Error("header_repository: error inserting receipt: ", err)
return receiptId, err
}
return receiptId, err
}
func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
var header core.Header
err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
@@ -216,26 +216,40 @@ var _ = Describe("Block header repository", func() {
Rlp: []byte{1, 2, 3},
}
_, receiptErr := repo.CreateReceiptInTx(tx, headerID, txId, receipt)
receiptRepo := repositories.HeaderSyncReceiptRepository{}
_, receiptErr := receiptRepo.CreateHeaderSyncReceiptInTx(headerID, txId, receipt, tx)
Expect(receiptErr).ToNot(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).ToNot(HaveOccurred())
Expect(receiptErr).ToNot(HaveOccurred())
type idModel struct {
TransactionId int64 `db:"transaction_id"`
core.Receipt
TransactionId int64 `db:"transaction_id"`
ContractAddressId int64 `db:"contract_address_id"`
CumulativeGasUsed uint64 `db:"cumulative_gas_used"`
GasUsed uint64 `db:"gas_used"`
StateRoot string `db:"state_root"`
Status int
TxHash string `db:"tx_hash"`
Rlp []byte `db:"rlp"`
}
var addressId int64
getAddressErr := db.Get(&addressId, `SELECT id FROM addresses WHERE address = $1`, contractAddr.Hex())
Expect(getAddressErr).NotTo(HaveOccurred())
var dbReceipt idModel
err = db.Get(&dbReceipt,
`SELECT transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
getReceiptErr := db.Get(&dbReceipt,
`SELECT transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
Expect(err).NotTo(HaveOccurred())
Expect(getReceiptErr).NotTo(HaveOccurred())
Expect(dbReceipt.TransactionId).To(Equal(txId))
Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
Expect(dbReceipt.ContractAddress).To(Equal(contractAddr.Hex()))
Expect(dbReceipt.ContractAddressId).To(Equal(addressId))
Expect(dbReceipt.CumulativeGasUsed).To(Equal(uint64(100)))
Expect(dbReceipt.GasUsed).To(Equal(uint64(10)))
Expect(dbReceipt.StateRoot).To(Equal(stateRoot.Hex()))
Expect(dbReceipt.Status).To(Equal(0))
Expect(dbReceipt.Rlp).To(Equal([]byte{1, 2, 3}))
})
})
@@ -0,0 +1,46 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories
import (
"github.com/ethereum/go-ethereum/log"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/pkg/core"
)
type HeaderSyncReceiptRepository struct{}
func (HeaderSyncReceiptRepository) CreateHeaderSyncReceiptInTx(headerID, transactionID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
var receiptId int64
addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
if getAddressErr != nil {
log.Error("createReceipt: Error getting address id: ", getAddressErr)
return receiptId, getAddressErr
}
err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
(header_id, transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, transaction_id) DO UPDATE
SET (contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9)
RETURNING id`,
headerID, transactionID, addressId, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId)
if err != nil {
log.Error("header_repository: error inserting receipt: ", err)
return receiptId, err
}
return receiptId, err
}
@@ -0,0 +1,133 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package repositories_test
import (
"encoding/json"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/test_config"
)
var _ = Describe("Header Sync Receipt Repo", func() {
var (
rawHeader []byte
err error
timestamp string
db *postgres.DB
receiptRepo repositories.HeaderSyncReceiptRepository
headerRepo repositories.HeaderRepository
header core.Header
)
BeforeEach(func() {
rawHeader, err = json.Marshal(types.Header{})
Expect(err).NotTo(HaveOccurred())
timestamp = big.NewInt(123456789).String()
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
receiptRepo = repositories.HeaderSyncReceiptRepository{}
headerRepo = repositories.NewHeaderRepository(db)
header = core.Header{
BlockNumber: 100,
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
Raw: rawHeader,
Timestamp: timestamp,
}
})
Describe("creating a receipt", func() {
It("adds a receipt in a tx", func() {
headerID, err := headerRepo.CreateOrUpdateHeader(header)
Expect(err).NotTo(HaveOccurred())
fromAddress := common.HexToAddress("0x1234")
toAddress := common.HexToAddress("0x5678")
txHash := common.HexToHash("0x9876")
txIndex := big.NewInt(123)
transaction := core.TransactionModel{
Data: []byte{},
From: fromAddress.Hex(),
GasLimit: 0,
GasPrice: 0,
Hash: txHash.Hex(),
Nonce: 0,
Raw: []byte{},
To: toAddress.Hex(),
TxIndex: txIndex.Int64(),
Value: "0",
}
tx, err := db.Beginx()
Expect(err).ToNot(HaveOccurred())
txId, txErr := headerRepo.CreateTransactionInTx(tx, headerID, transaction)
Expect(txErr).ToNot(HaveOccurred())
contractAddr := common.HexToAddress("0x1234")
stateRoot := common.HexToHash("0x5678")
receipt := core.Receipt{
ContractAddress: contractAddr.Hex(),
TxHash: txHash.Hex(),
GasUsed: 10,
CumulativeGasUsed: 100,
StateRoot: stateRoot.Hex(),
Rlp: []byte{1, 2, 3},
}
_, receiptErr := receiptRepo.CreateHeaderSyncReceiptInTx(headerID, txId, receipt, tx)
Expect(receiptErr).ToNot(HaveOccurred())
commitErr := tx.Commit()
Expect(commitErr).ToNot(HaveOccurred())
type idModel struct {
TransactionId int64 `db:"transaction_id"`
ContractAddressId int64 `db:"contract_address_id"`
CumulativeGasUsed uint64 `db:"cumulative_gas_used"`
GasUsed uint64 `db:"gas_used"`
StateRoot string `db:"state_root"`
Status int
TxHash string `db:"tx_hash"`
Rlp []byte `db:"rlp"`
}
var addressId int64
getAddressErr := db.Get(&addressId, `SELECT id FROM addresses WHERE address = $1`, contractAddr.Hex())
Expect(getAddressErr).NotTo(HaveOccurred())
var dbReceipt idModel
getReceiptErr := db.Get(&dbReceipt,
`SELECT transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
Expect(getReceiptErr).NotTo(HaveOccurred())
Expect(dbReceipt.TransactionId).To(Equal(txId))
Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
Expect(dbReceipt.ContractAddressId).To(Equal(addressId))
Expect(dbReceipt.CumulativeGasUsed).To(Equal(uint64(100)))
Expect(dbReceipt.GasUsed).To(Equal(uint64(10)))
Expect(dbReceipt.StateRoot).To(Equal(stateRoot.Hex()))
Expect(dbReceipt.Status).To(Equal(0))
Expect(dbReceipt.Rlp).To(Equal([]byte{1, 2, 3}))
})
})
})
@@ -34,7 +34,7 @@ var _ = Describe("Logs Repository", func() {
var db *postgres.DB
var blockRepository datastore.BlockRepository
var logsRepository datastore.LogRepository
var receiptRepository datastore.ReceiptRepository
var receiptRepository datastore.FullSyncReceiptRepository
var node core.Node
BeforeEach(func() {
@@ -48,14 +48,16 @@ var _ = Describe("Logs Repository", func() {
test_config.CleanTestDB(db)
blockRepository = repositories.NewBlockRepository(db)
logsRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
})
It("returns the log when it exists", func() {
blockNumber := int64(12345)
blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
Expect(err).NotTo(HaveOccurred())
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
tx, _ := db.Beginx()
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
tx.Commit()
Expect(err).NotTo(HaveOccurred())
err = logsRepository.CreateLogs([]core.Log{{
BlockNumber: blockNumber,
@@ -91,7 +93,9 @@ var _ = Describe("Logs Repository", func() {
blockNumber := int64(12345)
blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
Expect(err).NotTo(HaveOccurred())
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
tx, _ := db.Beginx()
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
tx.Commit()
Expect(err).NotTo(HaveOccurred())
err = logsRepository.CreateLogs([]core.Log{{
@@ -19,6 +19,7 @@ package repositories_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -32,7 +33,7 @@ var _ = Describe("Watched Events Repository", func() {
var blocksRepository datastore.BlockRepository
var filterRepository datastore.FilterRepository
var logRepository datastore.LogRepository
var receiptRepository datastore.ReceiptRepository
var receiptRepository datastore.FullSyncReceiptRepository
var watchedEventRepository datastore.WatchedEventRepository
BeforeEach(func() {
@@ -41,7 +42,7 @@ var _ = Describe("Watched Events Repository", func() {
blocksRepository = repositories.NewBlockRepository(db)
filterRepository = repositories.FilterRepository{DB: db}
logRepository = repositories.LogRepository{DB: db}
receiptRepository = repositories.ReceiptRepository{DB: db}
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
watchedEventRepository = repositories.WatchedEventRepository{DB: db}
})
@@ -79,7 +80,10 @@ var _ = Describe("Watched Events Repository", func() {
Expect(err).ToNot(HaveOccurred())
blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{})
Expect(err).NotTo(HaveOccurred())
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
tx, txBeginErr := db.Beginx()
Expect(txBeginErr).NotTo(HaveOccurred())
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
tx.Commit()
Expect(err).NotTo(HaveOccurred())
err = logRepository.CreateLogs(logs, receiptId)
Expect(err).ToNot(HaveOccurred())
@@ -136,7 +140,9 @@ var _ = Describe("Watched Events Repository", func() {
Expect(err).ToNot(HaveOccurred())
blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{Hash: "Ox123"})
Expect(err).NotTo(HaveOccurred())
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{TxHash: "0x123"})
tx, _ := db.Beginx()
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
tx.Commit()
Expect(err).NotTo(HaveOccurred())
err = logRepository.CreateLogs(logs, receiptId)
Expect(err).ToNot(HaveOccurred())
+12 -3
View File
@@ -17,10 +17,15 @@
package datastore
import (
"github.com/jmoiron/sqlx"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/filters"
)
type AddressRepository interface {
GetOrCreateAddress(address string) (int, error)
}
type BlockRepository interface {
CreateOrUpdateBlock(block core.Block) (int64, error)
GetBlock(blockNumber int64) (core.Block, error)
@@ -51,10 +56,14 @@ type LogRepository interface {
GetLogs(address string, blockNumber int64) ([]core.Log, error)
}
type ReceiptRepository interface {
type FullSyncReceiptRepository interface {
CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error
CreateReceipt(blockId int64, receipt core.Receipt) (int64, error)
GetReceipt(txHash string) (core.Receipt, error)
CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error)
GetFullSyncReceipt(txHash string) (core.Receipt, error)
}
type HeaderSyncReceiptRepository interface {
CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error)
}
type WatchedEventRepository interface {
+1 -1
View File
@@ -98,7 +98,7 @@ func (chain *MockBlockChain) GetHeaderByNumber(blockNumber int64) (core.Header,
return core.Header{BlockNumber: blockNumber}, nil
}
func (chain *MockBlockChain) GetHeaderByNumbers(blockNumbers []int64) ([]core.Header, error) {
func (chain *MockBlockChain) GetHeadersByNumbers(blockNumbers []int64) ([]core.Header, error) {
var headers []core.Header
for _, blockNumber := range blockNumbers {
var header = core.Header{BlockNumber: int64(blockNumber)}
+3 -2
View File
@@ -17,6 +17,7 @@
package fakes
import (
"github.com/jmoiron/sqlx"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -49,11 +50,11 @@ func (mrr *MockReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts
return mrr.createReceiptsAndLogsReturnErr
}
func (mrr *MockReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
func (mrr *MockReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
panic("implement me")
}
func (mrr *MockReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) {
func (mrr *MockReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) {
panic("implement me")
}
+6 -3
View File
@@ -79,7 +79,7 @@ func (blockChain *BlockChain) GetHeaderByNumber(blockNumber int64) (header core.
return blockChain.getPOWHeader(blockNumber)
}
func (blockChain *BlockChain) GetHeaderByNumbers(blockNumbers []int64) (header []core.Header, err error) {
func (blockChain *BlockChain) GetHeadersByNumbers(blockNumbers []int64) (header []core.Header, err error) {
if blockChain.node.NetworkID == core.KOVAN_NETWORK_ID {
return blockChain.getPOAHeaders(blockNumbers)
}
@@ -129,6 +129,9 @@ func (blockChain *BlockChain) GetTransactions(transactionHashes []common.Hash) (
func (blockChain *BlockChain) LastBlock() (*big.Int, error) {
block, err := blockChain.ethClient.HeaderByNumber(context.Background(), nil)
if err != nil {
return big.NewInt(0), err
}
return block.Number, err
}
@@ -159,7 +162,7 @@ func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Heade
Number: POAHeader.Number.ToInt(),
GasLimit: uint64(POAHeader.GasLimit),
GasUsed: uint64(POAHeader.GasUsed),
Time: POAHeader.Time.ToInt(),
Time: uint64(POAHeader.Time),
Extra: POAHeader.Extra,
}, POAHeader.Hash.String()), nil
}
@@ -208,7 +211,7 @@ func (blockChain *BlockChain) getPOAHeaders(blockNumbers []int64) (headers []cor
Number: POAHeader.Number.ToInt(),
GasLimit: uint64(POAHeader.GasLimit),
GasUsed: uint64(POAHeader.GasUsed),
Time: POAHeader.Time.ToInt(),
Time: uint64(POAHeader.Time),
Extra: POAHeader.Extra,
}, POAHeader.Hash.String())
+2 -2
View File
@@ -93,7 +93,7 @@ var _ = Describe("Geth blockchain", func() {
})
It("fetches headers with multiple blocks", func() {
_, err := blockChain.GetHeaderByNumbers([]int64{100, 99})
_, err := blockChain.GetHeadersByNumbers([]int64{100, 99})
Expect(err).NotTo(HaveOccurred())
mockRpcClient.AssertBatchCalledWith("eth_getBlockByNumber", 2)
@@ -139,7 +139,7 @@ var _ = Describe("Geth blockchain", func() {
blockNumber := hexutil.Big(*big.NewInt(100))
mockRpcClient.SetReturnPOAHeaders([]vulcCore.POAHeader{{Number: &blockNumber}})
_, err := blockChain.GetHeaderByNumbers([]int64{100, 99})
_, err := blockChain.GetHeadersByNumbers([]int64{100, 99})
Expect(err).NotTo(HaveOccurred())
mockRpcClient.AssertBatchCalledWith("eth_getBlockByNumber", 2)
+2 -2
View File
@@ -26,10 +26,10 @@ type ColdImporter struct {
blockRepository datastore.BlockRepository
converter common.BlockConverter
ethDB ethereum.Database
receiptRepository datastore.ReceiptRepository
receiptRepository datastore.FullSyncReceiptRepository
}
func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.ReceiptRepository, converter common.BlockConverter) *ColdImporter {
func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.FullSyncReceiptRepository, converter common.BlockConverter) *ColdImporter {
return &ColdImporter{
blockRepository: blockRepository,
converter: converter,
@@ -19,6 +19,7 @@ package common
import (
"encoding/json"
"math/big"
"strconv"
"strings"
"github.com/ethereum/go-ethereum/common/hexutil"
@@ -52,7 +53,7 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error)
Number: gethBlock.Number().Int64(),
ParentHash: gethBlock.ParentHash().Hex(),
Size: gethBlock.Size().String(),
Time: gethBlock.Time().Int64(),
Time: gethBlock.Time(),
Transactions: transactions,
UncleHash: gethBlock.UncleHash().Hex(),
}
@@ -81,7 +82,7 @@ func (bc BlockConverter) ToCoreUncle(block core.Block, uncles []*types.Header) (
Hash: uncle.Hash().Hex(),
Raw: raw,
Reward: thisUncleReward.String(),
Timestamp: uncle.Time.String(),
Timestamp: strconv.FormatUint(uncle.Time, 10),
}
coreUncles = append(coreUncles, coreUncle)
totalUncleRewards.Add(totalUncleRewards, thisUncleReward)
@@ -44,7 +44,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
extraData, _ := hexutil.Decode("0xe4b883e5bda9e7a59ee4bb99e9b1bc")
nonce := types.BlockNonce{10}
number := int64(1)
time := int64(140000000)
time := uint64(140000000)
header := types.Header{
Difficulty: difficulty,
@@ -55,7 +55,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
Nonce: nonce,
Number: big.NewInt(number),
ParentHash: common.Hash{64},
Time: big.NewInt(time),
Time: time,
UncleHash: common.Hash{128},
}
block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
@@ -18,6 +18,7 @@ package common
import (
"encoding/json"
"strconv"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -34,7 +35,7 @@ func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash str
Hash: blockHash,
BlockNumber: gethHeader.Number.Int64(),
Raw: rawHeader,
Timestamp: gethHeader.Time.String(),
Timestamp: strconv.FormatUint(gethHeader.Time, 10),
}
return coreHeader
}
@@ -19,6 +19,7 @@ package common_test
import (
"encoding/json"
"math/big"
"strconv"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@@ -37,7 +38,7 @@ var _ = Describe("Block header converter", func() {
ParentHash: common.HexToHash("0xParent"),
ReceiptHash: common.HexToHash("0xReceipt"),
Root: common.HexToHash("0xRoot"),
Time: big.NewInt(123456789),
Time: uint64(123456789),
TxHash: common.HexToHash("0xTransaction"),
UncleHash: common.HexToHash("0xUncle"),
}
@@ -48,7 +49,7 @@ var _ = Describe("Block header converter", func() {
Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
Expect(coreHeader.Hash).To(Equal(hash))
Expect(coreHeader.Timestamp).To(Equal(gethHeader.Time.String()))
Expect(coreHeader.Timestamp).To(Equal(strconv.FormatUint(gethHeader.Time, 10)))
})
It("includes raw bytes for header as JSON", func() {
@@ -1,3 +1,19 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package rpc_test
import (
+6 -1
View File
@@ -17,6 +17,7 @@
package history
import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -35,7 +36,7 @@ func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore
return 0, nil
}
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
log.Debug(getBlockRangeString(blockRange))
_, err = RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
if err != nil {
log.Error("PopulateMissingBlocks: error gettings/updating blocks: ", err)
@@ -61,3 +62,7 @@ func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datasto
}
return len(blockNumbers), nil
}
func getBlockRangeString(blockRange []int64) string {
return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
}
+8 -8
View File
@@ -24,14 +24,14 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
)
func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
lastBlock, err := blockchain.LastBlock()
func PopulateMissingHeaders(blockChain core.BlockChain, headerRepository datastore.HeaderRepository, startingBlockNumber int64) (int, error) {
lastBlock, err := blockChain.LastBlock()
if err != nil {
log.Error("PopulateMissingHeaders: Error getting last block: ", err)
return 0, err
}
blockNumbers, err := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockchain.Node().ID)
blockNumbers, err := headerRepository.MissingBlockNumbers(startingBlockNumber, lastBlock.Int64(), blockChain.Node().ID)
if err != nil {
log.Error("PopulateMissingHeaders: Error getting missing block numbers: ", err)
return 0, err
@@ -39,17 +39,17 @@ func PopulateMissingHeaders(blockchain core.BlockChain, headerRepository datasto
return 0, nil
}
log.Printf("Backfilling %d blocks\n\n", len(blockNumbers))
_, err = RetrieveAndUpdateHeaders(blockchain, headerRepository, blockNumbers)
log.Debug(getBlockRangeString(blockNumbers))
_, err = RetrieveAndUpdateHeaders(blockChain, headerRepository, blockNumbers)
if err != nil {
log.Error("PopulateMissingHeaders: Error getting/updating headers:", err)
log.Error("PopulateMissingHeaders: Error getting/updating headers: ", err)
return 0, err
}
return len(blockNumbers), nil
}
func RetrieveAndUpdateHeaders(chain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
headers, err := chain.GetHeaderByNumbers(blockNumbers)
func RetrieveAndUpdateHeaders(blockChain core.BlockChain, headerRepository datastore.HeaderRepository, blockNumbers []int64) (int, error) {
headers, err := blockChain.GetHeadersByNumbers(blockNumbers)
for _, header := range headers {
_, err = headerRepository.CreateOrUpdateHeader(header)
if err != nil {

Some files were not shown because too many files have changed in this diff Show More