forked from cerc-io/ipld-eth-server
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6a2a278ce | ||
|
|
edc0bdf668 | ||
|
|
1b3786338f | ||
|
|
58c23c6632 | ||
|
|
38f6847ff2 | ||
|
|
7e38764618 | ||
|
|
4e40e892d2 | ||
|
|
bcd6d14fcd | ||
|
|
258035833b | ||
|
|
4b61c87b55 | ||
|
|
1373fe83a1 | ||
|
|
b13eefbce7 | ||
|
|
3005a1ad50 | ||
|
|
750754caa7 | ||
|
|
e093ad351f | ||
|
|
33e7651096 | ||
|
|
baea9740c1 | ||
|
|
4f8b057726 | ||
|
|
65e762f80b | ||
|
|
dc8d52a58d | ||
|
|
f2547b5a42 | ||
|
|
62e454b7fe | ||
|
|
ca4451d914 | ||
|
|
f51f5c64a0 | ||
|
|
987abd4b2e | ||
|
|
a3b55bfd56 | ||
|
|
ee77fc6521 | ||
|
|
85aba29cf2 | ||
|
|
2c66afbcd7 | ||
|
|
a188e1dd79 | ||
|
|
92d153b010 | ||
|
|
d7a82e353b | ||
|
|
69ad521482 | ||
|
|
d30b4ea4f2 | ||
|
|
51915f7506 | ||
|
|
ebfb4965c7 | ||
|
|
b0fff9a1dd | ||
|
|
2c092e8d04 | ||
|
|
af6190ea09 | ||
|
|
0f95267e84 | ||
|
|
f0bcf930f4 | ||
|
|
0558792282 | ||
|
|
62cca62432 | ||
|
|
7092d33338 | ||
|
|
11a65c525c | ||
|
|
3d57f6030d | ||
|
|
b124bea974 |
+3
-3
@@ -1,7 +1,7 @@
|
||||
dist: trusty
|
||||
language: go
|
||||
go:
|
||||
- 1.11
|
||||
- 1.12
|
||||
services:
|
||||
- postgresql
|
||||
addons:
|
||||
@@ -17,8 +17,8 @@ before_script:
|
||||
- sudo -u postgres createdb vulcanize_private
|
||||
- make migrate NAME=vulcanize_private
|
||||
script:
|
||||
- make test
|
||||
- make integrationtest
|
||||
- env GO111MODULE=on make test
|
||||
- env GO111MODULE=on make integrationtest
|
||||
notifications:
|
||||
email: false
|
||||
env:
|
||||
|
||||
Generated
-563
@@ -1,563 +0,0 @@
|
||||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
digest = "1:48a213e9dc4880bbbd6999309a476fa4d3cc67560aa7127154cf8ea95bd464c2"
|
||||
name = "github.com/allegro/bigcache"
|
||||
packages = [
|
||||
".",
|
||||
"queue",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "f31987a23e44c5121ef8c8b2f2ea2e8ffa37b068"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
|
||||
name = "github.com/aristanetworks/goarista"
|
||||
packages = ["monotime"]
|
||||
pruneopts = ""
|
||||
revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
|
||||
name = "github.com/btcsuite/btcd"
|
||||
packages = ["btcec"]
|
||||
pruneopts = ""
|
||||
revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:5d47691333460db6ac83ced03c79b4bdb9aff3e322be24affb7855bed8affc6c"
|
||||
name = "github.com/dave/jennifer"
|
||||
packages = ["jen"]
|
||||
pruneopts = ""
|
||||
revision = "14e399b6b5e8456c66c45c955fc27b568bacb5c9"
|
||||
version = "v1.3.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
|
||||
name = "github.com/deckarep/golang-set"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
|
||||
version = "v1.7.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:90d36f5b581e95e00ced808cd48824ed6c320c25887828cce461bdef4cb7bc7c"
|
||||
name = "github.com/ethereum/go-ethereum"
|
||||
packages = [
|
||||
".",
|
||||
"accounts",
|
||||
"accounts/abi",
|
||||
"accounts/abi/bind",
|
||||
"accounts/keystore",
|
||||
"common",
|
||||
"common/bitutil",
|
||||
"common/hexutil",
|
||||
"common/math",
|
||||
"common/mclock",
|
||||
"common/prque",
|
||||
"core/rawdb",
|
||||
"core/types",
|
||||
"crypto",
|
||||
"crypto/ecies",
|
||||
"crypto/secp256k1",
|
||||
"ethclient",
|
||||
"ethdb",
|
||||
"ethdb/leveldb",
|
||||
"ethdb/memorydb",
|
||||
"event",
|
||||
"log",
|
||||
"metrics",
|
||||
"p2p",
|
||||
"p2p/discover",
|
||||
"p2p/discv5",
|
||||
"p2p/enode",
|
||||
"p2p/enr",
|
||||
"p2p/nat",
|
||||
"p2p/netutil",
|
||||
"params",
|
||||
"rlp",
|
||||
"rpc",
|
||||
"trie",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
|
||||
name = "github.com/fsnotify/fsnotify"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
|
||||
version = "v1.4.7"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
|
||||
name = "github.com/go-stack/stack"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
|
||||
version = "v1.8.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
|
||||
name = "github.com/golang/protobuf"
|
||||
packages = ["proto"]
|
||||
pruneopts = ""
|
||||
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
|
||||
name = "github.com/golang/snappy"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
|
||||
name = "github.com/google/uuid"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
|
||||
name = "github.com/hashicorp/golang-lru"
|
||||
packages = [
|
||||
".",
|
||||
"simplelru",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
|
||||
name = "github.com/hashicorp/hcl"
|
||||
packages = [
|
||||
".",
|
||||
"hcl/ast",
|
||||
"hcl/parser",
|
||||
"hcl/printer",
|
||||
"hcl/scanner",
|
||||
"hcl/strconv",
|
||||
"hcl/token",
|
||||
"json/parser",
|
||||
"json/scanner",
|
||||
"json/token",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
|
||||
name = "github.com/hpcloud/tail"
|
||||
packages = [
|
||||
".",
|
||||
"ratelimiter",
|
||||
"util",
|
||||
"watch",
|
||||
"winfile",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
|
||||
name = "github.com/huin/goupnp"
|
||||
packages = [
|
||||
".",
|
||||
"dcps/internetgateway1",
|
||||
"dcps/internetgateway2",
|
||||
"httpu",
|
||||
"scpd",
|
||||
"soap",
|
||||
"ssdp",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
|
||||
name = "github.com/inconshreveable/mousetrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
|
||||
version = "v1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
|
||||
name = "github.com/jackpal/go-nat-pmp"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
|
||||
name = "github.com/jmoiron/sqlx"
|
||||
packages = [
|
||||
".",
|
||||
"reflectx",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3"
|
||||
name = "github.com/konsorten/go-windows-terminal-sequences"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
|
||||
name = "github.com/lib/pq"
|
||||
packages = [
|
||||
".",
|
||||
"oid",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
|
||||
name = "github.com/magiconair/properties"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "c2353362d570a7bfa228149c62842019201cfb71"
|
||||
version = "v1.8.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
|
||||
name = "github.com/mitchellh/go-homedir"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
|
||||
name = "github.com/mitchellh/mapstructure"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
|
||||
name = "github.com/onsi/ginkgo"
|
||||
packages = [
|
||||
".",
|
||||
"config",
|
||||
"internal/codelocation",
|
||||
"internal/containernode",
|
||||
"internal/failer",
|
||||
"internal/leafnodes",
|
||||
"internal/remote",
|
||||
"internal/spec",
|
||||
"internal/spec_iterator",
|
||||
"internal/specrunner",
|
||||
"internal/suite",
|
||||
"internal/testingtproxy",
|
||||
"internal/writer",
|
||||
"reporters",
|
||||
"reporters/stenographer",
|
||||
"reporters/stenographer/support/go-colorable",
|
||||
"reporters/stenographer/support/go-isatty",
|
||||
"types",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
|
||||
version = "v1.6.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
|
||||
name = "github.com/onsi/gomega"
|
||||
packages = [
|
||||
".",
|
||||
"format",
|
||||
"ghttp",
|
||||
"internal/assertion",
|
||||
"internal/asyncassertion",
|
||||
"internal/oraclematcher",
|
||||
"internal/testingtsupport",
|
||||
"matchers",
|
||||
"matchers/support/goraph/bipartitegraph",
|
||||
"matchers/support/goraph/edge",
|
||||
"matchers/support/goraph/node",
|
||||
"matchers/support/goraph/util",
|
||||
"types",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
|
||||
version = "v1.4.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
|
||||
name = "github.com/pborman/uuid"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
|
||||
version = "v1.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
|
||||
name = "github.com/pelletier/go-toml"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
|
||||
name = "github.com/rjeczalik/notify"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
|
||||
version = "v0.9.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
|
||||
name = "github.com/rs/cors"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
|
||||
version = "v1.5.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:9d57e200ef5ccc4217fe0a34287308bac652435e7c6513f6263e0493d2245c56"
|
||||
name = "github.com/sirupsen/logrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
|
||||
name = "github.com/spf13/afero"
|
||||
packages = [
|
||||
".",
|
||||
"mem",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
|
||||
version = "v1.1.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
|
||||
name = "github.com/spf13/cast"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8965335b8c7107321228e3e3702cab9832751bac"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
|
||||
name = "github.com/spf13/cobra"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
|
||||
version = "v0.0.3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
|
||||
name = "github.com/spf13/jwalterweatherman"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
|
||||
version = "v1.0.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
|
||||
name = "github.com/spf13/viper"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
|
||||
name = "github.com/syndtr/goleveldb"
|
||||
packages = [
|
||||
"leveldb",
|
||||
"leveldb/cache",
|
||||
"leveldb/comparer",
|
||||
"leveldb/errors",
|
||||
"leveldb/filter",
|
||||
"leveldb/iterator",
|
||||
"leveldb/journal",
|
||||
"leveldb/memdb",
|
||||
"leveldb/opt",
|
||||
"leveldb/storage",
|
||||
"leveldb/table",
|
||||
"leveldb/util",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:59b49c47c11a48f1054529207f65907c014ecf5f9a7c0d9c0f1616dec7b062ed"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = [
|
||||
"pbkdf2",
|
||||
"scrypt",
|
||||
"sha3",
|
||||
"ssh/terminal",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"context",
|
||||
"html",
|
||||
"html/atom",
|
||||
"html/charset",
|
||||
"websocket",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
|
||||
name = "golang.org/x/sync"
|
||||
packages = ["errgroup"]
|
||||
pruneopts = ""
|
||||
revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
|
||||
name = "golang.org/x/sys"
|
||||
packages = [
|
||||
"unix",
|
||||
"windows",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
|
||||
name = "golang.org/x/text"
|
||||
packages = [
|
||||
"encoding",
|
||||
"encoding/charmap",
|
||||
"encoding/htmlindex",
|
||||
"encoding/internal",
|
||||
"encoding/internal/identifier",
|
||||
"encoding/japanese",
|
||||
"encoding/korean",
|
||||
"encoding/simplifiedchinese",
|
||||
"encoding/traditionalchinese",
|
||||
"encoding/unicode",
|
||||
"internal/gen",
|
||||
"internal/tag",
|
||||
"internal/triegen",
|
||||
"internal/ucd",
|
||||
"internal/utf8internal",
|
||||
"language",
|
||||
"runes",
|
||||
"transform",
|
||||
"unicode/cldr",
|
||||
"unicode/norm",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
|
||||
name = "gopkg.in/fsnotify.v1"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
|
||||
source = "gopkg.in/fsnotify/fsnotify.v1"
|
||||
version = "v1.4.7"
|
||||
|
||||
[[projects]]
|
||||
branch = "v2"
|
||||
digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
|
||||
name = "gopkg.in/natefinch/npipe.v2"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
|
||||
|
||||
[[projects]]
|
||||
branch = "v1"
|
||||
digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
|
||||
name = "gopkg.in/tomb.v1"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
|
||||
name = "gopkg.in/yaml.v2"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
|
||||
version = "v2.2.1"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
input-imports = [
|
||||
"github.com/dave/jennifer/jen",
|
||||
"github.com/ethereum/go-ethereum",
|
||||
"github.com/ethereum/go-ethereum/accounts/abi",
|
||||
"github.com/ethereum/go-ethereum/accounts/abi/bind",
|
||||
"github.com/ethereum/go-ethereum/common",
|
||||
"github.com/ethereum/go-ethereum/common/hexutil",
|
||||
"github.com/ethereum/go-ethereum/core/rawdb",
|
||||
"github.com/ethereum/go-ethereum/core/types",
|
||||
"github.com/ethereum/go-ethereum/crypto",
|
||||
"github.com/ethereum/go-ethereum/ethclient",
|
||||
"github.com/ethereum/go-ethereum/ethdb",
|
||||
"github.com/ethereum/go-ethereum/p2p",
|
||||
"github.com/ethereum/go-ethereum/p2p/discv5",
|
||||
"github.com/ethereum/go-ethereum/rlp",
|
||||
"github.com/ethereum/go-ethereum/rpc",
|
||||
"github.com/hashicorp/golang-lru",
|
||||
"github.com/hpcloud/tail",
|
||||
"github.com/jmoiron/sqlx",
|
||||
"github.com/lib/pq",
|
||||
"github.com/mitchellh/go-homedir",
|
||||
"github.com/onsi/ginkgo",
|
||||
"github.com/onsi/gomega",
|
||||
"github.com/onsi/gomega/ghttp",
|
||||
"github.com/sirupsen/logrus",
|
||||
"github.com/spf13/cobra",
|
||||
"github.com/spf13/viper",
|
||||
"golang.org/x/net/context",
|
||||
"golang.org/x/sync/errgroup",
|
||||
"gopkg.in/tomb.v1",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
|
||||
# Gopkg.toml example
|
||||
#
|
||||
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
|
||||
# for detailed Gopkg.toml documentation.
|
||||
#
|
||||
# required = ["github.com/user/thing/cmd/thing"]
|
||||
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project"
|
||||
# version = "1.0.0"
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project2"
|
||||
# branch = "dev"
|
||||
# source = "github.com/myfork/project2"
|
||||
#
|
||||
# [[override]]
|
||||
# name = "github.com/x/y"
|
||||
# version = "2.4.0"
|
||||
|
||||
|
||||
[[override]]
|
||||
name = "gopkg.in/fsnotify.v1"
|
||||
source = "gopkg.in/fsnotify/fsnotify.v1"
|
||||
|
||||
[[override]]
|
||||
name = "github.com/pressly/sup"
|
||||
version = "0.5.3"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/onsi/ginkgo"
|
||||
version = "1.4.0"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/jmoiron/sqlx"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/lib/pq"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/sirupsen/logrus"
|
||||
version = "1.2.0"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/spf13/cobra"
|
||||
version = "0.0.1"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/ethereum/go-ethereum"
|
||||
revision = "cd79bc61a983d6482579d12cdd239b37bbfa12ef"
|
||||
@@ -3,11 +3,6 @@ BASE = $(GOPATH)/src/$(PACKAGE)
|
||||
PKGS = go list ./... | grep -v "^vendor/"
|
||||
|
||||
# Tools
|
||||
## Dependency management
|
||||
DEP = $(BIN)/dep
|
||||
$(BIN)/dep:
|
||||
go get -u github.com/golang/dep/cmd/dep
|
||||
|
||||
## Testing library
|
||||
GINKGO = $(BIN)/ginkgo
|
||||
$(BIN)/ginkgo:
|
||||
@@ -32,7 +27,7 @@ $(BIN)/gometalinter.v2:
|
||||
|
||||
|
||||
.PHONY: installtools
|
||||
installtools: | $(LINT) $(GOOSE) $(GINKGO) $(DEP)
|
||||
installtools: | $(LINT) $(GOOSE) $(GINKGO)
|
||||
echo "Installing tools"
|
||||
|
||||
.PHONY: metalint
|
||||
@@ -58,11 +53,7 @@ integrationtest: | $(GINKGO) $(LINT)
|
||||
go fmt ./...
|
||||
$(GINKGO) -r integration_test/
|
||||
|
||||
.PHONY: dep
|
||||
dep: | $(DEP)
|
||||
$(DEP) ensure
|
||||
|
||||
build: dep
|
||||
build:
|
||||
go fmt ./...
|
||||
go build
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ data from VulcanizeDB's underlying Postgres database and making it accessible.
|
||||
1. [Configuring a synced Ethereum node](#configuring-a-synced-ethereum-node)
|
||||
|
||||
### Dependencies
|
||||
- Go 1.11+
|
||||
- Go 1.12+
|
||||
- Postgres 11.2
|
||||
- Ethereum Node
|
||||
- [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.23+)
|
||||
@@ -46,16 +46,17 @@ Download the codebase to your local `GOPATH` via:
|
||||
|
||||
`go get github.com/vulcanize/vulcanizedb`
|
||||
|
||||
Move to the project directory and use [golang/dep](https://github.com/golang/dep) to install the dependencies:
|
||||
Move to the project directory:
|
||||
|
||||
`cd $GOPATH/src/github.com/vulcanize/vulcanizedb`
|
||||
|
||||
`dep ensure`
|
||||
|
||||
Once the dependencies have been successfully installed, build the executable with:
|
||||
Be sure you have enabled Go Modules (`export GO111MODULE=on`), and build the executable with:
|
||||
|
||||
`make build`
|
||||
|
||||
If you need to use a different dependency than what is currently defined in `go.mod`, it may helpful to look into [the replace directive](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive).
|
||||
This instruction enables you to point at a fork or the local filesystem for dependency resolution.
|
||||
|
||||
If you are running into issues at this stage, ensure that `GOPATH` is defined in your shell.
|
||||
If necessary, `GOPATH` can be set in `~/.bashrc` or `~/.bash_profile`, depending upon your system.
|
||||
It can be additionally helpful to add `$GOPATH/bin` to your shell's `$PATH`.
|
||||
|
||||
+8
-6
@@ -39,6 +39,8 @@ var coldImportCmd = &cobra.Command{
|
||||
|
||||
Geth must be synced over all of the desired blocks and must not be running in order to execute this command.`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
coldImport()
|
||||
},
|
||||
}
|
||||
@@ -55,7 +57,7 @@ func coldImport() {
|
||||
ethDBConfig := ethereum.CreateDatabaseConfig(ethereum.Level, levelDbPath)
|
||||
ethDB, err := ethereum.CreateDatabase(ethDBConfig)
|
||||
if err != nil {
|
||||
log.Fatal("Error connecting to ethereum db: ", err)
|
||||
LogWithCommand.Fatal("Error connecting to ethereum db: ", err)
|
||||
}
|
||||
mostRecentBlockNumberInDb := ethDB.GetHeadBlockNumber()
|
||||
if syncAll {
|
||||
@@ -63,10 +65,10 @@ func coldImport() {
|
||||
endingBlockNumber = mostRecentBlockNumberInDb
|
||||
}
|
||||
if endingBlockNumber < startingBlockNumber {
|
||||
log.Fatal("Ending block number must be greater than starting block number for cold import.")
|
||||
LogWithCommand.Fatal("Ending block number must be greater than starting block number for cold import.")
|
||||
}
|
||||
if endingBlockNumber > mostRecentBlockNumberInDb {
|
||||
log.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
|
||||
LogWithCommand.Fatal("Ending block number is greater than most recent block in db: ", mostRecentBlockNumberInDb)
|
||||
}
|
||||
|
||||
// init pg db
|
||||
@@ -76,13 +78,13 @@ func coldImport() {
|
||||
nodeBuilder := cold_import.NewColdImportNodeBuilder(reader, parser)
|
||||
coldNode, err := nodeBuilder.GetNode(genesisBlock, levelDbPath)
|
||||
if err != nil {
|
||||
log.Fatal("Error getting node: ", err)
|
||||
LogWithCommand.Fatal("Error getting node: ", err)
|
||||
}
|
||||
pgDB := utils.LoadPostgres(databaseConfig, coldNode)
|
||||
|
||||
// init cold importer deps
|
||||
blockRepository := repositories.NewBlockRepository(&pgDB)
|
||||
receiptRepository := repositories.ReceiptRepository{DB: &pgDB}
|
||||
receiptRepository := repositories.FullSyncReceiptRepository{DB: &pgDB}
|
||||
transactionConverter := cold_db.NewColdDbTransactionConverter()
|
||||
blockConverter := vulcCommon.NewBlockConverter(transactionConverter)
|
||||
|
||||
@@ -90,6 +92,6 @@ func coldImport() {
|
||||
coldImporter := cold_import.NewColdImporter(ethDB, blockRepository, receiptRepository, blockConverter)
|
||||
err = coldImporter.Execute(startingBlockNumber, endingBlockNumber, coldNode.ID)
|
||||
if err != nil {
|
||||
log.Fatal("Error executing cold import: ", err)
|
||||
LogWithCommand.Fatal("Error executing cold import: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
+24
-19
@@ -1,17 +1,18 @@
|
||||
// Copyright © 2019 Vulcanize, Inc
|
||||
//
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
@@ -101,6 +102,8 @@ single config file or in separate command instances using different config files
|
||||
Specify config location when executing the command:
|
||||
./vulcanizedb compose --config=./environments/config_name.toml`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
compose()
|
||||
},
|
||||
}
|
||||
@@ -110,23 +113,25 @@ func compose() {
|
||||
prepConfig()
|
||||
|
||||
// Generate code to build the plugin according to the config file
|
||||
log.Info("generating plugin")
|
||||
LogWithCommand.Info("generating plugin")
|
||||
generator, err := p2.NewGenerator(genConfig, databaseConfig)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Debug("initializing plugin generator failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
err = generator.GenerateExporterPlugin()
|
||||
if err != nil {
|
||||
log.Debug("generating plugin failed")
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Debug("generating plugin failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
// TODO: Embed versioning info in the .so files so we know which version of vulcanizedb to run them with
|
||||
_, pluginPath, err := genConfig.GetPluginPaths()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Debug("getting plugin path failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
fmt.Printf("Composed plugin %s", pluginPath)
|
||||
log.Info("plugin .so file output to", pluginPath)
|
||||
LogWithCommand.Info("plugin .so file output to ", pluginPath)
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -134,38 +139,38 @@ func init() {
|
||||
}
|
||||
|
||||
func prepConfig() {
|
||||
log.Info("configuring plugin")
|
||||
LogWithCommand.Info("configuring plugin")
|
||||
names := viper.GetStringSlice("exporter.transformerNames")
|
||||
transformers := make(map[string]config.Transformer)
|
||||
for _, name := range names {
|
||||
transformer := viper.GetStringMapString("exporter." + name)
|
||||
p, pOK := transformer["path"]
|
||||
if !pOK || p == "" {
|
||||
log.Fatal(name, "transformer config is missing `path` value")
|
||||
LogWithCommand.Fatal(name, " transformer config is missing `path` value")
|
||||
}
|
||||
r, rOK := transformer["repository"]
|
||||
if !rOK || r == "" {
|
||||
log.Fatal(name, "transformer config is missing `repository` value")
|
||||
LogWithCommand.Fatal(name, " transformer config is missing `repository` value")
|
||||
}
|
||||
m, mOK := transformer["migrations"]
|
||||
if !mOK || m == "" {
|
||||
log.Fatal(name, "transformer config is missing `migrations` value")
|
||||
LogWithCommand.Fatal(name, " transformer config is missing `migrations` value")
|
||||
}
|
||||
mr, mrOK := transformer["rank"]
|
||||
if !mrOK || mr == "" {
|
||||
log.Fatal(name, "transformer config is missing `rank` value")
|
||||
LogWithCommand.Fatal(name, " transformer config is missing `rank` value")
|
||||
}
|
||||
rank, err := strconv.ParseUint(mr, 10, 64)
|
||||
if err != nil {
|
||||
log.Fatal(name, "migration `rank` can't be converted to an unsigned integer")
|
||||
LogWithCommand.Fatal(name, " migration `rank` can't be converted to an unsigned integer")
|
||||
}
|
||||
t, tOK := transformer["type"]
|
||||
if !tOK {
|
||||
log.Fatal(name, "transformer config is missing `type` value")
|
||||
LogWithCommand.Fatal(name, " transformer config is missing `type` value")
|
||||
}
|
||||
transformerType := config.GetTransformerType(t)
|
||||
if transformerType == config.UnknownTransformerType {
|
||||
log.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
|
||||
LogWithCommand.Fatal(errors.New(`unknown transformer type in exporter config accepted types are "eth_event", "eth_storage"`))
|
||||
}
|
||||
|
||||
transformers[name] = config.Transformer{
|
||||
|
||||
+20
-17
@@ -1,17 +1,18 @@
|
||||
// Copyright © 2019 Vulcanize, Inc
|
||||
//
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
@@ -105,6 +106,8 @@ single config file or in separate command instances using different config files
|
||||
Specify config location when executing the command:
|
||||
./vulcanizedb composeAndExecute --config=./environments/config_name.toml`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
composeAndExecute()
|
||||
},
|
||||
}
|
||||
@@ -114,44 +117,44 @@ func composeAndExecute() {
|
||||
prepConfig()
|
||||
|
||||
// Generate code to build the plugin according to the config file
|
||||
log.Info("generating plugin")
|
||||
LogWithCommand.Info("generating plugin")
|
||||
generator, err := p2.NewGenerator(genConfig, databaseConfig)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
err = generator.GenerateExporterPlugin()
|
||||
if err != nil {
|
||||
log.Debug("generating plugin failed")
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Debug("generating plugin failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// Get the plugin path and load the plugin
|
||||
_, pluginPath, err := genConfig.GetPluginPaths()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
if !genConfig.Save {
|
||||
defer helpers.ClearFiles(pluginPath)
|
||||
}
|
||||
log.Info("linking plugin", pluginPath)
|
||||
LogWithCommand.Info("linking plugin ", pluginPath)
|
||||
plug, err := plugin.Open(pluginPath)
|
||||
if err != nil {
|
||||
log.Debug("linking plugin failed")
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Debug("linking plugin failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// Load the `Exporter` symbol from the plugin
|
||||
log.Info("loading transformers from plugin")
|
||||
LogWithCommand.Info("loading transformers from plugin")
|
||||
symExporter, err := plug.Lookup("Exporter")
|
||||
if err != nil {
|
||||
log.Debug("loading Exporter symbol failed")
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Debug("loading Exporter symbol failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// Assert that the symbol is of type Exporter
|
||||
exporter, ok := symExporter.(Exporter)
|
||||
if !ok {
|
||||
log.Debug("plugged-in symbol not of type Exporter")
|
||||
LogWithCommand.Debug("plugged-in symbol not of type Exporter")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
|
||||
@@ -79,6 +79,8 @@ Requires a .toml config file:
|
||||
piping = true
|
||||
`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
contractWatcher()
|
||||
},
|
||||
}
|
||||
@@ -103,18 +105,18 @@ func contractWatcher() {
|
||||
case "full":
|
||||
t = ft.NewTransformer(con, blockChain, &db)
|
||||
default:
|
||||
log.Fatal("Invalid mode")
|
||||
LogWithCommand.Fatal("Invalid mode")
|
||||
}
|
||||
|
||||
err := t.Init()
|
||||
if err != nil {
|
||||
log.Fatal(fmt.Sprintf("Failed to initialized transformer\r\nerr: %v\r\n", err))
|
||||
LogWithCommand.Fatal(fmt.Sprintf("Failed to initialize transformer, err: %v ", err))
|
||||
}
|
||||
|
||||
for range ticker.C {
|
||||
err = t.Execute()
|
||||
if err != nil {
|
||||
log.Error("Execution error for transformer:", t.GetConfig().Name, err)
|
||||
LogWithCommand.Error("Execution error for transformer: ", t.GetConfig().Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+19
-18
@@ -1,23 +1,23 @@
|
||||
// Copyright © 2019 Vulcanize, Inc
|
||||
//
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"plugin"
|
||||
syn "sync"
|
||||
"time"
|
||||
@@ -64,6 +64,8 @@ must have been composed by the same version of vulcanizedb or else it will not b
|
||||
Specify config location when executing the command:
|
||||
./vulcanizedb execute --config=./environments/config_name.toml`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
execute()
|
||||
},
|
||||
}
|
||||
@@ -75,30 +77,29 @@ func execute() {
|
||||
// Get the plugin path and load the plugin
|
||||
_, pluginPath, err := genConfig.GetPluginPaths()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("Executing plugin %s", pluginPath)
|
||||
log.Info("linking plugin", pluginPath)
|
||||
LogWithCommand.Info("linking plugin ", pluginPath)
|
||||
plug, err := plugin.Open(pluginPath)
|
||||
if err != nil {
|
||||
log.Debug("linking plugin failed")
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Warn("linking plugin failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// Load the `Exporter` symbol from the plugin
|
||||
log.Info("loading transformers from plugin")
|
||||
LogWithCommand.Info("loading transformers from plugin")
|
||||
symExporter, err := plug.Lookup("Exporter")
|
||||
if err != nil {
|
||||
log.Debug("loading Exporter symbol failed")
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Warn("loading Exporter symbol failed")
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// Assert that the symbol is of type Exporter
|
||||
exporter, ok := symExporter.(Exporter)
|
||||
if !ok {
|
||||
log.Debug("plugged-in symbol not of type Exporter")
|
||||
os.Exit(1)
|
||||
LogWithCommand.Fatal("plugged-in symbol not of type Exporter")
|
||||
}
|
||||
|
||||
// Use the Exporters export method to load the EventTransformerInitializer, StorageTransformerInitializer, and ContractTransformerInitializer sets
|
||||
@@ -149,7 +150,7 @@ type Exporter interface {
|
||||
func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
|
||||
defer wg.Done()
|
||||
// Execute over the EventTransformerInitializer set using the watcher
|
||||
log.Info("executing event transformers")
|
||||
LogWithCommand.Info("executing event transformers")
|
||||
var recheck constants.TransformerExecution
|
||||
if recheckHeadersArg {
|
||||
recheck = constants.HeaderRecheck
|
||||
@@ -166,7 +167,7 @@ func watchEthEvents(w *watcher.EventWatcher, wg *syn.WaitGroup) {
|
||||
func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
|
||||
defer wg.Done()
|
||||
// Execute over the StorageTransformerInitializer set using the storage watcher
|
||||
log.Info("executing storage transformers")
|
||||
LogWithCommand.Info("executing storage transformers")
|
||||
ticker := time.NewTicker(pollingInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
@@ -179,7 +180,7 @@ func watchEthStorage(w *watcher.StorageWatcher, wg *syn.WaitGroup) {
|
||||
func watchEthContract(w *watcher.ContractWatcher, wg *syn.WaitGroup) {
|
||||
defer wg.Done()
|
||||
// Execute over the ContractTransformerInitializer set using the contract watcher
|
||||
log.Info("executing contract_watcher transformers")
|
||||
LogWithCommand.Info("executing contract_watcher transformers")
|
||||
ticker := time.NewTicker(pollingInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
|
||||
+8
-7
@@ -49,20 +49,21 @@ Expects ethereum node to be running and requires a .toml config:
|
||||
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
|
||||
`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
fullSync()
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(fullSyncCmd)
|
||||
|
||||
fullSyncCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block number to start syncing from")
|
||||
}
|
||||
|
||||
func backFillAllBlocks(blockchain core.BlockChain, blockRepository datastore.BlockRepository, missingBlocksPopulated chan int, startingBlockNumber int64) {
|
||||
populated, err := history.PopulateMissingBlocks(blockchain, blockRepository, startingBlockNumber)
|
||||
if err != nil {
|
||||
log.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
|
||||
LogWithCommand.Error("backfillAllBlocks: error in populateMissingBlocks: ", err)
|
||||
}
|
||||
missingBlocksPopulated <- populated
|
||||
}
|
||||
@@ -74,13 +75,13 @@ func fullSync() {
|
||||
blockChain := getBlockChain()
|
||||
lastBlock, err := blockChain.LastBlock()
|
||||
if err != nil {
|
||||
log.Error("fullSync: Error getting last block: ", err)
|
||||
LogWithCommand.Error("fullSync: Error getting last block: ", err)
|
||||
}
|
||||
if lastBlock.Int64() == 0 {
|
||||
log.Fatal("geth initial: state sync not finished")
|
||||
LogWithCommand.Fatal("geth initial: state sync not finished")
|
||||
}
|
||||
if startingBlockNumber > lastBlock.Int64() {
|
||||
log.Fatal("fullSync: starting block number > current block number")
|
||||
LogWithCommand.Fatal("fullSync: starting block number > current block number")
|
||||
}
|
||||
|
||||
db := utils.LoadPostgres(databaseConfig, blockChain.Node())
|
||||
@@ -94,9 +95,9 @@ func fullSync() {
|
||||
case <-ticker.C:
|
||||
window, err := validator.ValidateBlocks()
|
||||
if err != nil {
|
||||
log.Error("fullSync: error in validateBlocks: ", err)
|
||||
LogWithCommand.Error("fullSync: error in validateBlocks: ", err)
|
||||
}
|
||||
log.Info(window.GetString())
|
||||
LogWithCommand.Debug(window.GetString())
|
||||
case <-missingBlocksPopulated:
|
||||
go backFillAllBlocks(blockChain, blockRepository, missingBlocksPopulated, startingBlockNumber)
|
||||
}
|
||||
|
||||
+8
-6
@@ -50,6 +50,8 @@ Expects ethereum node to be running and requires a .toml config:
|
||||
ipcPath = "/Users/user/Library/Ethereum/geth.ipc"
|
||||
`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
SubCommand = cmd.CalledAs()
|
||||
LogWithCommand = *log.WithField("SubCommand", SubCommand)
|
||||
headerSync()
|
||||
},
|
||||
}
|
||||
@@ -64,7 +66,7 @@ func backFillAllHeaders(blockchain core.BlockChain, headerRepository datastore.H
|
||||
if err != nil {
|
||||
// TODO Lots of possible errors in the call stack above. If errors occur, we still put
|
||||
// 0 in the channel, triggering another round
|
||||
log.Error("backfillAllHeaders: Error populating headers: ", err)
|
||||
LogWithCommand.Error("backfillAllHeaders: Error populating headers: ", err)
|
||||
}
|
||||
missingBlocksPopulated <- populated
|
||||
}
|
||||
@@ -86,9 +88,9 @@ func headerSync() {
|
||||
case <-ticker.C:
|
||||
window, err := validator.ValidateHeaders()
|
||||
if err != nil {
|
||||
log.Error("headerSync: ValidateHeaders failed: ", err)
|
||||
LogWithCommand.Error("headerSync: ValidateHeaders failed: ", err)
|
||||
}
|
||||
log.Info(window.GetString())
|
||||
LogWithCommand.Debug(window.GetString())
|
||||
case n := <-missingBlocksPopulated:
|
||||
if n == 0 {
|
||||
time.Sleep(3 * time.Second)
|
||||
@@ -101,12 +103,12 @@ func headerSync() {
|
||||
func validateArgs(blockChain *geth.BlockChain) {
|
||||
lastBlock, err := blockChain.LastBlock()
|
||||
if err != nil {
|
||||
log.Error("validateArgs: Error getting last block: ", err)
|
||||
LogWithCommand.Error("validateArgs: Error getting last block: ", err)
|
||||
}
|
||||
if lastBlock.Int64() == 0 {
|
||||
log.Fatal("geth initial: state sync not finished")
|
||||
LogWithCommand.Fatal("geth initial: state sync not finished")
|
||||
}
|
||||
if startingBlockNumber > lastBlock.Int64() {
|
||||
log.Fatal("starting block number > current block number")
|
||||
LogWithCommand.Fatal("starting block number > current block number")
|
||||
}
|
||||
}
|
||||
|
||||
+31
-9
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -47,6 +46,8 @@ var (
|
||||
syncAll bool
|
||||
endingBlockNumber int64
|
||||
recheckHeadersArg bool
|
||||
SubCommand string
|
||||
LogWithCommand log.Entry
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -56,18 +57,26 @@ const (
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "vulcanizedb",
|
||||
PersistentPreRun: database,
|
||||
PersistentPreRun: initFuncs,
|
||||
}
|
||||
|
||||
func Execute() {
|
||||
log.Info("----- Starting vDB -----")
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
log.Fatal(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func database(cmd *cobra.Command, args []string) {
|
||||
func initFuncs(cmd *cobra.Command, args []string) {
|
||||
setViperConfigs()
|
||||
logLvlErr := logLevel()
|
||||
if logLvlErr != nil {
|
||||
log.Fatal("Could not set log level: ", logLvlErr)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func setViperConfigs() {
|
||||
ipc = viper.GetString("client.ipcpath")
|
||||
levelDbPath = viper.GetString("client.leveldbpath")
|
||||
storageDiffsPath = viper.GetString("filesystem.storageDiffsPath")
|
||||
@@ -81,6 +90,19 @@ func database(cmd *cobra.Command, args []string) {
|
||||
viper.Set("database.config", databaseConfig)
|
||||
}
|
||||
|
||||
func logLevel() error {
|
||||
lvl, err := log.ParseLevel(viper.GetString("log.level"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.SetLevel(lvl)
|
||||
if lvl > log.InfoLevel {
|
||||
log.SetReportCaller(true)
|
||||
}
|
||||
log.Info("Log level set to ", lvl.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
cobra.OnInitialize(initConfig)
|
||||
// When searching for env variables, replace dots in config keys with underscores
|
||||
@@ -97,6 +119,7 @@ func init() {
|
||||
rootCmd.PersistentFlags().String("client-levelDbPath", "", "location of levelDb chaindata")
|
||||
rootCmd.PersistentFlags().String("filesystem-storageDiffsPath", "", "location of storage diffs csv file")
|
||||
rootCmd.PersistentFlags().String("exporter-name", "exporter", "name of exporter plugin")
|
||||
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
|
||||
|
||||
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
|
||||
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
|
||||
@@ -107,6 +130,7 @@ func init() {
|
||||
viper.BindPFlag("client.levelDbPath", rootCmd.PersistentFlags().Lookup("client-levelDbPath"))
|
||||
viper.BindPFlag("filesystem.storageDiffsPath", rootCmd.PersistentFlags().Lookup("filesystem-storageDiffsPath"))
|
||||
viper.BindPFlag("exporter.fileName", rootCmd.PersistentFlags().Lookup("exporter-name"))
|
||||
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
@@ -116,16 +140,14 @@ func initConfig() {
|
||||
noConfigError := "No config file passed with --config flag"
|
||||
fmt.Println("Error: ", noConfigError)
|
||||
log.Fatal(noConfigError)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := viper.ReadInConfig(); err == nil {
|
||||
log.Printf("Using config file: %s\n\n", viper.ConfigFileUsed())
|
||||
} else {
|
||||
invalidConfigError := "Couldn't read config file"
|
||||
fmt.Println("Error: ", invalidConfigError)
|
||||
log.Fatal(invalidConfigError)
|
||||
os.Exit(1)
|
||||
formattedError := fmt.Sprintf("%s: %s", invalidConfigError, err.Error())
|
||||
log.Fatal(formattedError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +155,7 @@ func getBlockChain() *geth.BlockChain {
|
||||
rawRpcClient, err := rpc.Dial(ipc)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
LogWithCommand.Fatal(err)
|
||||
}
|
||||
rpcClient := client.NewRpcClient(rawRpcClient, ipc)
|
||||
ethClient := ethclient.NewClient(rawRpcClient)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE public.addresses
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
address character varying(42),
|
||||
UNIQUE (address)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE public.addresses;
|
||||
@@ -1,16 +0,0 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE full_sync_receipts
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
|
||||
contract_address VARCHAR(42),
|
||||
cumulative_gas_used NUMERIC,
|
||||
gas_used NUMERIC,
|
||||
state_root VARCHAR(66),
|
||||
status INTEGER,
|
||||
tx_hash VARCHAR(66)
|
||||
);
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE full_sync_receipts;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE full_sync_receipts
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
transaction_id INTEGER NOT NULL REFERENCES full_sync_transactions (id) ON DELETE CASCADE,
|
||||
contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
|
||||
cumulative_gas_used NUMERIC,
|
||||
gas_used NUMERIC,
|
||||
state_root VARCHAR(66),
|
||||
status INTEGER,
|
||||
tx_hash VARCHAR(66)
|
||||
);
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE full_sync_receipts;
|
||||
@@ -1,18 +0,0 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE header_sync_receipts(
|
||||
id SERIAL PRIMARY KEY,
|
||||
transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions(id) ON DELETE CASCADE,
|
||||
header_id INTEGER NOT NULL REFERENCES headers(id) ON DELETE CASCADE,
|
||||
contract_address VARCHAR(42),
|
||||
cumulative_gas_used NUMERIC,
|
||||
gas_used NUMERIC,
|
||||
state_root VARCHAR(66),
|
||||
status INTEGER,
|
||||
tx_hash VARCHAR(66),
|
||||
rlp BYTEA,
|
||||
UNIQUE(header_id, transaction_id)
|
||||
);
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE header_sync_receipts;
|
||||
@@ -0,0 +1,19 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE header_sync_receipts
|
||||
(
|
||||
id SERIAL PRIMARY KEY,
|
||||
transaction_id INTEGER NOT NULL REFERENCES header_sync_transactions (id) ON DELETE CASCADE,
|
||||
header_id INTEGER NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
|
||||
contract_address_id INTEGER NOT NULL REFERENCES addresses (id) ON DELETE CASCADE,
|
||||
cumulative_gas_used NUMERIC,
|
||||
gas_used NUMERIC,
|
||||
state_root VARCHAR(66),
|
||||
status INTEGER,
|
||||
tx_hash VARCHAR(66),
|
||||
rlp BYTEA,
|
||||
UNIQUE (header_id, transaction_id)
|
||||
);
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE header_sync_receipts;
|
||||
+71
-2
@@ -20,6 +20,36 @@ SET default_tablespace = '';
|
||||
|
||||
SET default_with_oids = false;
|
||||
|
||||
--
|
||||
-- Name: addresses; Type: TABLE; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
CREATE TABLE public.addresses (
|
||||
id integer NOT NULL,
|
||||
address character varying(42)
|
||||
);
|
||||
|
||||
|
||||
--
|
||||
-- Name: addresses_id_seq; Type: SEQUENCE; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
CREATE SEQUENCE public.addresses_id_seq
|
||||
AS integer
|
||||
START WITH 1
|
||||
INCREMENT BY 1
|
||||
NO MINVALUE
|
||||
NO MAXVALUE
|
||||
CACHE 1;
|
||||
|
||||
|
||||
--
|
||||
-- Name: addresses_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id;
|
||||
|
||||
|
||||
--
|
||||
-- Name: logs; Type: TABLE; Schema: public; Owner: -
|
||||
--
|
||||
@@ -144,7 +174,7 @@ CREATE TABLE public.eth_nodes (
|
||||
|
||||
CREATE TABLE public.full_sync_receipts (
|
||||
id integer NOT NULL,
|
||||
contract_address character varying(42),
|
||||
contract_address_id integer NOT NULL,
|
||||
cumulative_gas_used numeric,
|
||||
gas_used numeric,
|
||||
state_root character varying(66),
|
||||
@@ -254,7 +284,7 @@ CREATE TABLE public.header_sync_receipts (
|
||||
id integer NOT NULL,
|
||||
transaction_id integer NOT NULL,
|
||||
header_id integer NOT NULL,
|
||||
contract_address character varying(42),
|
||||
contract_address_id integer NOT NULL,
|
||||
cumulative_gas_used numeric,
|
||||
gas_used numeric,
|
||||
state_root character varying(66),
|
||||
@@ -564,6 +594,13 @@ CREATE VIEW public.watched_event_logs AS
|
||||
WHERE ((((log_filters.topic0)::text = (logs.topic0)::text) OR (log_filters.topic0 IS NULL)) AND (((log_filters.topic1)::text = (logs.topic1)::text) OR (log_filters.topic1 IS NULL)) AND (((log_filters.topic2)::text = (logs.topic2)::text) OR (log_filters.topic2 IS NULL)) AND (((log_filters.topic3)::text = (logs.topic3)::text) OR (log_filters.topic3 IS NULL)));
|
||||
|
||||
|
||||
--
|
||||
-- Name: addresses id; Type: DEFAULT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass);
|
||||
|
||||
|
||||
--
|
||||
-- Name: blocks id; Type: DEFAULT; Schema: public; Owner: -
|
||||
--
|
||||
@@ -662,6 +699,22 @@ ALTER TABLE ONLY public.uncles ALTER COLUMN id SET DEFAULT nextval('public.uncle
|
||||
ALTER TABLE ONLY public.watched_contracts ALTER COLUMN contract_id SET DEFAULT nextval('public.watched_contracts_contract_id_seq'::regclass);
|
||||
|
||||
|
||||
--
|
||||
-- Name: addresses addresses_address_key; Type: CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY public.addresses
|
||||
ADD CONSTRAINT addresses_address_key UNIQUE (address);
|
||||
|
||||
|
||||
--
|
||||
-- Name: addresses addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY public.addresses
|
||||
ADD CONSTRAINT addresses_pkey PRIMARY KEY (id);
|
||||
|
||||
|
||||
--
|
||||
-- Name: blocks blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
@@ -896,6 +949,14 @@ ALTER TABLE ONLY public.checked_headers
|
||||
ADD CONSTRAINT checked_headers_header_id_fkey FOREIGN KEY (header_id) REFERENCES public.headers(id) ON DELETE CASCADE;
|
||||
|
||||
|
||||
--
|
||||
-- Name: full_sync_receipts full_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY public.full_sync_receipts
|
||||
ADD CONSTRAINT full_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
|
||||
|
||||
|
||||
--
|
||||
-- Name: full_sync_transactions full_sync_transactions_block_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
@@ -904,6 +965,14 @@ ALTER TABLE ONLY public.full_sync_transactions
|
||||
ADD CONSTRAINT full_sync_transactions_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON DELETE CASCADE;
|
||||
|
||||
|
||||
--
|
||||
-- Name: header_sync_receipts header_sync_receipts_contract_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY public.header_sync_receipts
|
||||
ADD CONSTRAINT header_sync_receipts_contract_address_id_fkey FOREIGN KEY (contract_address_id) REFERENCES public.addresses(id) ON DELETE CASCADE;
|
||||
|
||||
|
||||
--
|
||||
-- Name: header_sync_receipts header_sync_receipts_header_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
|
||||
--
|
||||
|
||||
@@ -39,7 +39,7 @@ or [contract](../../staging/libraries/shared/watcher/contract_watcher.go#L68) wa
|
||||
* If the base vDB migrations occupy this path as well, they need to be in their `goose fix`ed form
|
||||
as they are [here](../../staging/db/migrations)
|
||||
|
||||
To update a plugin repository with changes to the core vulcanizedb repository, run `dep ensure` to update its dependencies.
|
||||
To update a plugin repository with changes to the core vulcanizedb repository, use your dependency manager to install the desired version of vDB.
|
||||
|
||||
## Building and Running Custom Transformers
|
||||
### Commands
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Repository Maintenance
|
||||
|
||||
## Diagrams
|
||||
- Diagrams were created with [draw.io](draw.io).
|
||||
- To update a diagram:
|
||||
1. Go to [draw.io](draw.io).
|
||||
1. Click on *File > Open from* and choose the location of the diagram you want to update.
|
||||
1. Once open in draw.io, you may update it.
|
||||
1. Export the diagram to this repository's directory and add commit it.
|
||||
|
||||
|
||||
## Generating the Changelog
|
||||
We use [github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) to
|
||||
generate release Changelogs. To be consistent with previous Changelogs, the following flags should be passed to the
|
||||
command:
|
||||
|
||||
```
|
||||
--user vulcanize
|
||||
--project vulcanizedb
|
||||
--token {YOUR_GITHUB_TOKEN}
|
||||
--no-issues
|
||||
--usernames-as-github-logins
|
||||
--since-tag {PREVIOUS_RELEASE_TAG}
|
||||
```
|
||||
|
||||
More information on why your github token is needed, and how to generate it here:
|
||||
https://github.com/github-changelog-generator/github-changelog-generator#github-token
|
||||
@@ -0,0 +1,33 @@
|
||||
module github.com/vulcanize/vulcanizedb
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/allegro/bigcache v1.2.1 // indirect
|
||||
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 // indirect
|
||||
github.com/dave/jennifer v1.3.0
|
||||
github.com/ethereum/go-ethereum v1.9.1
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/hpcloud/tail v1.0.0
|
||||
github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a
|
||||
github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 // indirect
|
||||
github.com/lib/pq v1.0.0
|
||||
github.com/mattn/go-isatty v0.0.9 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/onsi/ginkgo v1.7.0
|
||||
github.com/onsi/gomega v1.4.3
|
||||
github.com/pborman/uuid v1.2.0 // indirect
|
||||
github.com/pressly/goose v2.6.0+incompatible
|
||||
github.com/prometheus/tsdb v0.10.0 // indirect
|
||||
github.com/rs/cors v1.7.0 // indirect
|
||||
github.com/sirupsen/logrus v1.2.0
|
||||
github.com/spf13/cobra v0.0.3
|
||||
github.com/spf13/viper v1.3.2
|
||||
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 // indirect
|
||||
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
|
||||
github.com/tyler-smith/go-bip39 v1.0.2 // indirect
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
|
||||
)
|
||||
@@ -0,0 +1,296 @@
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
|
||||
github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc=
|
||||
github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
|
||||
github.com/apilayer/freegeoip v3.5.0+incompatible h1:z1u2gv0/rsSi/HqMDB436AiUROXXim7st5DOg4Ikl4A=
|
||||
github.com/apilayer/freegeoip v3.5.0+incompatible/go.mod h1:CUfFqErhFhXneJendyQ/rRcuA8kH8JxHvYnbOozmlCU=
|
||||
github.com/aristanetworks/goarista v0.0.0-20180907105523-ff33da284e76/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
|
||||
github.com/aristanetworks/goarista v0.0.0-20190121184617-8f049bdb8feb/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
|
||||
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
|
||||
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/btcsuite/btcd v0.0.0-20180903232927-cff30e1d23fc/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
|
||||
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw=
|
||||
github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU=
|
||||
github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
|
||||
github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
|
||||
github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
|
||||
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
|
||||
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
|
||||
github.com/elastic/gosigar v0.10.4 h1:6jfw75dsoflhBMRdO6QPzQUgLqUYTsQQQRkkcsHsuPo=
|
||||
github.com/elastic/gosigar v0.10.4/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
|
||||
github.com/ethereum/go-ethereum v1.9.0/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
|
||||
github.com/ethereum/go-ethereum v1.9.1 h1:MrdTRvKIa3apdx6NW1azzSgl8BQB1eTBVSUmFhuztaU=
|
||||
github.com/ethereum/go-ethereum v1.9.1/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
|
||||
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
|
||||
github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI=
|
||||
github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
|
||||
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
|
||||
github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20190610161739-8f92f34fc598/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6 h1:9WiNlI9Cds5S5YITwRpRs8edNaq0nxTEymhDW20A1QE=
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20190724201507-010347b5f9e6/go.mod h1:Au3iQ8DvDis8hZ4q2OzRcaKYlAsPt+fYvib5q4nIqu4=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY=
|
||||
github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
|
||||
github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
|
||||
github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
|
||||
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/influxdata/influxdb v1.7.7 h1:UvNzAPfBrKMENVbQ4mr4ccA9sW+W1Ihl0Yh1s0BiVAg=
|
||||
github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
|
||||
github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
|
||||
github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
|
||||
github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a h1:Jyg5PpIc1nLGrNDM5blVkiSySmRhaD/IiXkvaHzBYnw=
|
||||
github.com/jmoiron/sqlx v0.0.0-20181024163419-82935fac6c1a/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/karalabe/usb v0.0.0-20190703133951-9be757f914c0/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
|
||||
github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8 h1:VhnqxaTIudc9IWKx8uXRLnpdSb9noCEj+vHacjmhp68=
|
||||
github.com/karalabe/usb v0.0.0-20190819132248-550797b1cad8/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
|
||||
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
|
||||
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
|
||||
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo=
|
||||
github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
|
||||
github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
|
||||
github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
|
||||
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
|
||||
github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.9.1/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
|
||||
github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic=
|
||||
github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4=
|
||||
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
|
||||
github.com/rjeczalik/notify v0.9.2 h1:MiTWrPj55mNDHEiIX5YUSKefw/+lCQVoAFmD6oQm5w8=
|
||||
github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
|
||||
github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
|
||||
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
|
||||
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 h1:ju5UTwk5Odtm4trrY+4Ca4RMj5OyXbmVeDAVad2T0Jw=
|
||||
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q=
|
||||
github.com/steakknife/bloomfilter v0.0.0-20180906043351-99ee86d9200f/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
|
||||
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE=
|
||||
github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw=
|
||||
github.com/steakknife/hamming v0.0.0-20180906055317-003c143a81c2/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
|
||||
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM=
|
||||
github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
|
||||
github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 h1:GnOzE5fEFN3b2zDhJJABEofdb51uMRNb8eqIVtdducs=
|
||||
github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
|
||||
github.com/tyler-smith/go-bip39 v1.0.0/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
|
||||
github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8=
|
||||
github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/vulcanize/vulcanizedb v0.0.5/go.mod h1:utXkheCL9VjTfmuivuvRiAAyHh54GSN9XRQNEbFCA8k=
|
||||
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8cVcUy5CeIBYhEESkOB7m6Gmkrk=
|
||||
github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
|
||||
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff h1:uuol9OUzSvZntY1v963NAbVd7A+PHLMz1FlCe3Lorcs=
|
||||
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190709231704-1e4459ed25ff/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
|
||||
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
@@ -1,3 +1,19 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
|
||||
@@ -100,4 +100,54 @@ var _ = Describe("Storage transformer", func() {
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
|
||||
Describe("when a storage row contains more than one item packed in storage", func() {
|
||||
var (
|
||||
rawValue = common.HexToAddress("000000000000000000000000000000000000000000000002a300000000002a30")
|
||||
fakeBlockNumber = 123
|
||||
fakeBlockHash = "0x67890"
|
||||
packedTypes = make(map[int]utils.ValueType)
|
||||
)
|
||||
packedTypes[0] = utils.Uint48
|
||||
packedTypes[1] = utils.Uint48
|
||||
|
||||
var fakeMetadata = utils.StorageValueMetadata{
|
||||
Name: "",
|
||||
Keys: nil,
|
||||
Type: utils.PackedSlot,
|
||||
PackedTypes: packedTypes,
|
||||
}
|
||||
|
||||
It("passes the decoded data items to the repository", func() {
|
||||
mappings.Metadata = fakeMetadata
|
||||
fakeRow := utils.StorageDiffRow{
|
||||
Contract: common.Address{},
|
||||
BlockHash: common.HexToHash(fakeBlockHash),
|
||||
BlockHeight: fakeBlockNumber,
|
||||
StorageKey: common.Hash{},
|
||||
StorageValue: rawValue.Hash(),
|
||||
}
|
||||
|
||||
err := t.Execute(fakeRow)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(repository.PassedBlockNumber).To(Equal(fakeBlockNumber))
|
||||
Expect(repository.PassedBlockHash).To(Equal(common.HexToHash(fakeBlockHash).Hex()))
|
||||
Expect(repository.PassedMetadata).To(Equal(fakeMetadata))
|
||||
expectedPassedValue := make(map[int]string)
|
||||
expectedPassedValue[0] = "10800"
|
||||
expectedPassedValue[1] = "172800"
|
||||
Expect(repository.PassedValue.(map[int]string)).To(Equal(expectedPassedValue))
|
||||
})
|
||||
|
||||
It("returns error if creating a row fails", func() {
|
||||
mappings.Metadata = fakeMetadata
|
||||
repository.CreateErr = fakes.FakeError
|
||||
|
||||
err := t.Execute(utils.StorageDiffRow{StorageValue: rawValue.Hash()})
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(MatchError(fakes.FakeError))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -17,9 +17,12 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fs"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type IStorageFetcher interface {
|
||||
@@ -39,6 +42,7 @@ func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.S
|
||||
if tailErr != nil {
|
||||
errs <- tailErr
|
||||
}
|
||||
log.Debug("fetching storage diffs...")
|
||||
for line := range t.Lines {
|
||||
row, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
|
||||
if parseErr != nil {
|
||||
|
||||
@@ -76,7 +76,8 @@ func GetCheckedColumnNames(db *postgres.DB) ([]string, error) {
|
||||
`SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'checked_headers'
|
||||
AND column_name ~ '_checked';`
|
||||
AND column_name <> 'id'
|
||||
AND column_name <> 'header_id';`
|
||||
|
||||
err := db.Select(&queryResult, columnNamesQuery)
|
||||
if err != nil {
|
||||
|
||||
@@ -295,9 +295,9 @@ var _ = Describe("Repository", func() {
|
||||
|
||||
func getExpectedColumnNames() []string {
|
||||
return []string{
|
||||
"column_1_checked",
|
||||
"column_2_checked",
|
||||
"column_3_checked",
|
||||
"column_4_checked",
|
||||
"column_1",
|
||||
"column_2",
|
||||
"column_3",
|
||||
"column_4",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,27 +23,30 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
)
|
||||
|
||||
const (
|
||||
bitsPerByte = 8
|
||||
)
|
||||
|
||||
func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
|
||||
switch metadata.Type {
|
||||
case Uint256:
|
||||
return decodeUint256(row.StorageValue.Bytes()), nil
|
||||
return decodeInteger(row.StorageValue.Bytes()), nil
|
||||
case Uint48:
|
||||
return decodeUint48(row.StorageValue.Bytes()), nil
|
||||
return decodeInteger(row.StorageValue.Bytes()), nil
|
||||
case Uint128:
|
||||
return decodeInteger(row.StorageValue.Bytes()), nil
|
||||
case Address:
|
||||
return decodeAddress(row.StorageValue.Bytes()), nil
|
||||
case Bytes32:
|
||||
return row.StorageValue.Hex(), nil
|
||||
case PackedSlot:
|
||||
return decodePackedSlot(row.StorageValue.Bytes(), metadata.PackedTypes), nil
|
||||
default:
|
||||
panic(fmt.Sprintf("can't decode unknown type: %d", metadata.Type))
|
||||
}
|
||||
}
|
||||
|
||||
func decodeUint256(raw []byte) string {
|
||||
n := big.NewInt(0).SetBytes(raw)
|
||||
return n.String()
|
||||
}
|
||||
|
||||
func decodeUint48(raw []byte) string {
|
||||
func decodeInteger(raw []byte) string {
|
||||
n := big.NewInt(0).SetBytes(raw)
|
||||
return n.String()
|
||||
}
|
||||
@@ -51,3 +54,53 @@ func decodeUint48(raw []byte) string {
|
||||
func decodeAddress(raw []byte) string {
|
||||
return common.BytesToAddress(raw).Hex()
|
||||
}
|
||||
|
||||
func decodePackedSlot(raw []byte, packedTypes map[int]ValueType) map[int]string {
|
||||
storageSlotData := raw
|
||||
decodedStorageSlotItems := map[int]string{}
|
||||
numberOfTypes := len(packedTypes)
|
||||
|
||||
for position := 0; position < numberOfTypes; position++ {
|
||||
//get length of remaining storage date
|
||||
lengthOfStorageData := len(storageSlotData)
|
||||
|
||||
//get item details (type, length, starting index, value bytes)
|
||||
itemType := packedTypes[position]
|
||||
lengthOfItem := getNumberOfBytes(itemType)
|
||||
itemStartingIndex := lengthOfStorageData - lengthOfItem
|
||||
itemValueBytes := storageSlotData[itemStartingIndex:]
|
||||
|
||||
//decode item's bytes and set in results map
|
||||
decodedValue := decodeIndividualItem(itemValueBytes, itemType)
|
||||
decodedStorageSlotItems[position] = decodedValue
|
||||
|
||||
//pop last item off raw slot data before moving on
|
||||
storageSlotData = storageSlotData[0:itemStartingIndex]
|
||||
}
|
||||
|
||||
return decodedStorageSlotItems
|
||||
}
|
||||
|
||||
func decodeIndividualItem(itemBytes []byte, valueType ValueType) string {
|
||||
switch valueType {
|
||||
case Uint48, Uint128:
|
||||
return decodeInteger(itemBytes)
|
||||
case Address:
|
||||
return decodeAddress(itemBytes)
|
||||
default:
|
||||
panic(fmt.Sprintf("can't decode unknown type: %d", valueType))
|
||||
}
|
||||
}
|
||||
|
||||
func getNumberOfBytes(valueType ValueType) int {
|
||||
switch valueType {
|
||||
case Uint48:
|
||||
return 48 / bitsPerByte
|
||||
case Uint128:
|
||||
return 128 / bitsPerByte
|
||||
case Address:
|
||||
return 20
|
||||
default:
|
||||
panic(fmt.Sprintf("ValueType %d not recognized", valueType))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,17 @@ var _ = Describe("Storage decoder", func() {
|
||||
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
|
||||
})
|
||||
|
||||
It("decodes uint128", func() {
|
||||
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
|
||||
row := utils.StorageDiffRow{StorageValue: fakeInt}
|
||||
metadata := utils.StorageValueMetadata{Type: utils.Uint128}
|
||||
|
||||
result, err := utils.Decode(row, metadata)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
|
||||
})
|
||||
|
||||
It("decodes uint48", func() {
|
||||
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
|
||||
row := utils.StorageDiffRow{StorageValue: fakeInt}
|
||||
@@ -59,4 +70,105 @@ var _ = Describe("Storage decoder", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(Equal(fakeAddress.Hex()))
|
||||
})
|
||||
|
||||
Describe("when there are multiple items packed in the storage slot", func() {
|
||||
It("decodes uint48 items", func() {
|
||||
//this is a real storage data example
|
||||
packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
|
||||
row := utils.StorageDiffRow{StorageValue: packedStorage}
|
||||
packedTypes := map[int]utils.ValueType{}
|
||||
packedTypes[0] = utils.Uint48
|
||||
packedTypes[1] = utils.Uint48
|
||||
|
||||
metadata := utils.StorageValueMetadata{
|
||||
Type: utils.PackedSlot,
|
||||
PackedTypes: packedTypes,
|
||||
}
|
||||
|
||||
result, err := utils.Decode(row, metadata)
|
||||
decodedValues := result.(map[int]string)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
|
||||
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
|
||||
})
|
||||
|
||||
It("decodes 5 uint48 items", func() {
|
||||
//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
|
||||
//real storage data that has several items packed into it
|
||||
packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
|
||||
|
||||
packedStorage := common.HexToHash(packedStorageHex)
|
||||
row := utils.StorageDiffRow{StorageValue: packedStorage}
|
||||
packedTypes := map[int]utils.ValueType{}
|
||||
packedTypes[0] = utils.Uint48
|
||||
packedTypes[1] = utils.Uint48
|
||||
packedTypes[2] = utils.Uint48
|
||||
packedTypes[3] = utils.Uint48
|
||||
packedTypes[4] = utils.Uint48
|
||||
|
||||
metadata := utils.StorageValueMetadata{
|
||||
Type: utils.PackedSlot,
|
||||
PackedTypes: packedTypes,
|
||||
}
|
||||
|
||||
result, err := utils.Decode(row, metadata)
|
||||
decodedValues := result.(map[int]string)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
|
||||
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
|
||||
Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("9F3C6").Bytes()).String()))
|
||||
Expect(decodedValues[3]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("FFFFFFFFFFFE").Bytes()).String()))
|
||||
Expect(decodedValues[4]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("A5D1A").Bytes()).String()))
|
||||
})
|
||||
|
||||
It("decodes 2 uint128 items", func() {
|
||||
//TODO: this packedStorageHex was generated by hand, it would be nice to test this against
|
||||
//real storage data that has several items packed into it
|
||||
packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
|
||||
"0000000000000000AB54A98CEB1F0AD2"
|
||||
packedStorage := common.HexToHash(packedStorageHex)
|
||||
row := utils.StorageDiffRow{StorageValue: packedStorage}
|
||||
packedTypes := map[int]utils.ValueType{}
|
||||
packedTypes[0] = utils.Uint128
|
||||
packedTypes[1] = utils.Uint128
|
||||
|
||||
metadata := utils.StorageValueMetadata{
|
||||
Type: utils.PackedSlot,
|
||||
PackedTypes: packedTypes,
|
||||
}
|
||||
|
||||
result, err := utils.Decode(row, metadata)
|
||||
decodedValues := result.(map[int]string)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(decodedValues[0]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("AB54A98CEB1F0AD2").Bytes()).String()))
|
||||
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("38D7EA4C67FF8E502B6730000").Bytes()).String()))
|
||||
})
|
||||
|
||||
It("decodes address + 2 uint48s", func() {
|
||||
//TODO: replace with real data when available
|
||||
addressHex := "0000000000000000000000000000000000012345"
|
||||
packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
|
||||
row := utils.StorageDiffRow{StorageValue: packedStorage}
|
||||
packedTypes := map[int]utils.ValueType{}
|
||||
packedTypes[0] = utils.Address
|
||||
packedTypes[1] = utils.Uint48
|
||||
packedTypes[2] = utils.Uint48
|
||||
|
||||
metadata := utils.StorageValueMetadata{
|
||||
Type: utils.PackedSlot,
|
||||
PackedTypes: packedTypes,
|
||||
}
|
||||
|
||||
result, err := utils.Decode(row, metadata)
|
||||
decodedValues := result.(map[int]string)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(decodedValues[0]).To(Equal("0x" + addressHex))
|
||||
Expect(decodedValues[1]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a30").Bytes()).String()))
|
||||
Expect(decodedValues[2]).To(Equal(big.NewInt(0).SetBytes(common.HexToHash("2a300").Bytes()).String()))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -16,27 +16,54 @@
|
||||
|
||||
package utils
|
||||
|
||||
import "fmt"
|
||||
|
||||
type ValueType int
|
||||
|
||||
const (
|
||||
Uint256 ValueType = iota
|
||||
Uint48
|
||||
Uint128
|
||||
Bytes32
|
||||
Address
|
||||
PackedSlot
|
||||
)
|
||||
|
||||
type Key string
|
||||
|
||||
type StorageValueMetadata struct {
|
||||
Name string
|
||||
Keys map[Key]string
|
||||
Type ValueType
|
||||
Name string
|
||||
Keys map[Key]string
|
||||
Type ValueType
|
||||
PackedNames map[int]string //zero indexed position in map => name of packed item
|
||||
PackedTypes map[int]ValueType //zero indexed position in map => type of packed item
|
||||
}
|
||||
|
||||
func GetStorageValueMetadata(name string, keys map[Key]string, t ValueType) StorageValueMetadata {
|
||||
func GetStorageValueMetadata(name string, keys map[Key]string, valueType ValueType) StorageValueMetadata {
|
||||
return getMetadata(name, keys, valueType, nil, nil)
|
||||
}
|
||||
|
||||
func GetStorageValueMetadataForPackedSlot(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
|
||||
return getMetadata(name, keys, valueType, packedNames, packedTypes)
|
||||
}
|
||||
|
||||
func getMetadata(name string, keys map[Key]string, valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) StorageValueMetadata {
|
||||
assertPackedSlotArgs(valueType, packedNames, packedTypes)
|
||||
|
||||
return StorageValueMetadata{
|
||||
Name: name,
|
||||
Keys: keys,
|
||||
Type: t,
|
||||
Name: name,
|
||||
Keys: keys,
|
||||
Type: valueType,
|
||||
PackedNames: packedNames,
|
||||
PackedTypes: packedTypes,
|
||||
}
|
||||
}
|
||||
|
||||
func assertPackedSlotArgs(valueType ValueType, packedNames map[int]string, packedTypes map[int]ValueType) {
|
||||
if valueType == PackedSlot && (packedTypes == nil || packedNames == nil) {
|
||||
panic(fmt.Sprintf("ValueType is PackedSlot. Expected PackedNames and PackedTypes to not be nil, but got PackedNames = %v and PackedTypes = %v", packedNames, packedTypes))
|
||||
} else if (packedNames != nil && packedTypes != nil) && valueType != PackedSlot {
|
||||
panic(fmt.Sprintf("PackedNames and PackedTypes passed in. Expected ValueType to equal PackedSlot (%v), but got %v.", PackedSlot, valueType))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("Storage value metadata getter", func() {
|
||||
It("returns a storage value metadata instance with corresponding fields assigned", func() {
|
||||
It("returns storage value metadata for a single storage variable", func() {
|
||||
metadataName := "fake_name"
|
||||
metadataKeys := map[utils.Key]string{"key": "value"}
|
||||
metadataType := utils.Uint256
|
||||
@@ -19,4 +19,60 @@ var _ = Describe("Storage value metadata getter", func() {
|
||||
}
|
||||
Expect(utils.GetStorageValueMetadata(metadataName, metadataKeys, metadataType)).To(Equal(expectedMetadata))
|
||||
})
|
||||
|
||||
Describe("metadata for a packed storaged slot", func() {
|
||||
It("returns metadata for multiple storage variables", func() {
|
||||
metadataName := "fake_name"
|
||||
metadataKeys := map[utils.Key]string{"key": "value"}
|
||||
metadataType := utils.PackedSlot
|
||||
metadataPackedNames := map[int]string{0: "name"}
|
||||
metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
|
||||
|
||||
expectedMetadata := utils.StorageValueMetadata{
|
||||
Name: metadataName,
|
||||
Keys: metadataKeys,
|
||||
Type: metadataType,
|
||||
PackedTypes: metadataPackedTypes,
|
||||
PackedNames: metadataPackedNames,
|
||||
}
|
||||
Expect(utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)).To(Equal(expectedMetadata))
|
||||
})
|
||||
|
||||
It("panics if PackedTypes are nil when the type is PackedSlot", func() {
|
||||
metadataName := "fake_name"
|
||||
metadataKeys := map[utils.Key]string{"key": "value"}
|
||||
metadataType := utils.PackedSlot
|
||||
metadataPackedNames := map[int]string{0: "name"}
|
||||
|
||||
getMetadata := func() {
|
||||
utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, nil)
|
||||
}
|
||||
Expect(getMetadata).To(Panic())
|
||||
})
|
||||
|
||||
It("panics if PackedNames are nil when the type is PackedSlot", func() {
|
||||
metadataName := "fake_name"
|
||||
metadataKeys := map[utils.Key]string{"key": "value"}
|
||||
metadataType := utils.PackedSlot
|
||||
metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
|
||||
|
||||
getMetadata := func() {
|
||||
utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, nil, metadataPackedTypes)
|
||||
}
|
||||
Expect(getMetadata).To(Panic())
|
||||
})
|
||||
|
||||
It("panics if valueType is not PackedSlot if PackedNames is populated", func() {
|
||||
metadataName := "fake_name"
|
||||
metadataKeys := map[utils.Key]string{"key": "value"}
|
||||
metadataType := utils.Uint48
|
||||
metadataPackedNames := map[int]string{0: "name"}
|
||||
metadataPackedTypes := map[int]utils.ValueType{0: utils.Uint48}
|
||||
|
||||
getMetadata := func() {
|
||||
utils.GetStorageValueMetadataForPackedSlot(metadataName, metadataKeys, metadataType, metadataPackedNames, metadataPackedTypes)
|
||||
}
|
||||
Expect(getMetadata).To(Panic())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -100,7 +100,7 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
|
||||
|
||||
missingHeaders, err := repository.MissingHeaders(*watcher.StartingBlock, -1, watcher.DB, notCheckedSQL)
|
||||
if err != nil {
|
||||
logrus.Error("Fetching of missing headers failed in watcher!")
|
||||
logrus.Error("Couldn't fetch missing headers in watcher: ", err)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -108,7 +108,11 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
|
||||
// TODO Extend FetchLogs for doing several blocks at a time
|
||||
logs, err := watcher.Fetcher.FetchLogs(watcher.Addresses, watcher.Topics, header)
|
||||
if err != nil {
|
||||
logrus.Errorf("Error while fetching logs for header %v in watcher", header.Id)
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"headerId": header.Id,
|
||||
"headerHash": header.Hash,
|
||||
"blockNumber": header.BlockNumber,
|
||||
}).Errorf("Couldn't fetch logs for header: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -120,6 +124,7 @@ func (watcher *EventWatcher) Execute(recheckHeaders constants.TransformerExecuti
|
||||
|
||||
transformErr := watcher.transformLogs(logs, header)
|
||||
if transformErr != nil {
|
||||
logrus.Error("Could not transform logs: ", transformErr)
|
||||
return transformErr
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,15 @@ package main
|
||||
import (
|
||||
"github.com/vulcanize/vulcanizedb/cmd"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"os"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetFormatter(&log.JSONFormatter{})
|
||||
log.SetFormatter(&log.JSONFormatter{
|
||||
PrettyPrint: true,
|
||||
})
|
||||
file, err := os.OpenFile("vulcanizedb.log",
|
||||
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
|
||||
if err == nil {
|
||||
|
||||
@@ -17,7 +17,9 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
)
|
||||
|
||||
// Block retriever is used to retrieve the first block for a given contract and the most recent block
|
||||
@@ -41,7 +43,10 @@ func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
|
||||
func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
|
||||
i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
|
||||
if err != nil {
|
||||
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
|
||||
if err == sql.ErrNoRows {
|
||||
i, err = r.retrieveFirstBlockFromLogs(contractAddr)
|
||||
}
|
||||
return i, err
|
||||
}
|
||||
|
||||
return i, err
|
||||
@@ -49,18 +54,26 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
|
||||
|
||||
// For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
|
||||
func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
|
||||
var firstBlock int
|
||||
var firstBlock int64
|
||||
addressId, getAddressErr := addressRepository().GetOrCreateAddress(r.db, contractAddr)
|
||||
if getAddressErr != nil {
|
||||
return firstBlock, getAddressErr
|
||||
}
|
||||
err := r.db.Get(
|
||||
&firstBlock,
|
||||
`SELECT number FROM blocks
|
||||
WHERE id = (SELECT block_id FROM full_sync_receipts
|
||||
WHERE lower(contract_address) = $1
|
||||
WHERE contract_address_id = $1
|
||||
ORDER BY block_id ASC
|
||||
LIMIT 1)`,
|
||||
contractAddr,
|
||||
addressId,
|
||||
)
|
||||
|
||||
return int64(firstBlock), err
|
||||
return firstBlock, err
|
||||
}
|
||||
|
||||
func addressRepository() repositories.AddressRepository {
|
||||
return repositories.AddressRepository{}
|
||||
}
|
||||
|
||||
// In which case this servers as a heuristic to find the first block by finding the first contract event log
|
||||
|
||||
@@ -137,7 +137,7 @@ func SetupTusdRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string)
|
||||
}, core.Node{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
receiptRepository := repositories.ReceiptRepository{DB: db}
|
||||
receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
|
||||
logRepository := repositories.LogRepository{DB: db}
|
||||
blockRepository := *repositories.NewBlockRepository(db)
|
||||
|
||||
@@ -183,7 +183,7 @@ func SetupENSRepo(vulcanizeLogId *int64, wantedEvents, wantedMethods []string) (
|
||||
}, core.Node{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
receiptRepository := repositories.ReceiptRepository{DB: db}
|
||||
receiptRepository := repositories.FullSyncReceiptRepository{DB: db}
|
||||
logRepository := repositories.LogRepository{DB: db}
|
||||
blockRepository := *repositories.NewBlockRepository(db)
|
||||
|
||||
@@ -225,6 +225,9 @@ func TearDown(db *postgres.DB) {
|
||||
tx, err := db.Beginx()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = tx.Exec(`DELETE FROM addresses`)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
_, err = tx.Exec(`DELETE FROM blocks`)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
|
||||
@@ -220,7 +220,7 @@ var _ = Describe("Parser", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
selectMethods := p.GetMethods([]string{})
|
||||
Expect(len(selectMethods)).To(Equal(22))
|
||||
Expect(len(selectMethods)).To(Equal(25))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ type Block struct {
|
||||
Number int64 `db:"number"`
|
||||
ParentHash string `db:"parent_hash"`
|
||||
Size string `db:"size"`
|
||||
Time int64 `db:"time"`
|
||||
Time uint64 `db:"time"`
|
||||
Transactions []TransactionModel
|
||||
UncleHash string `db:"uncle_hash"`
|
||||
UnclesReward string `db:"uncles_reward"`
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ type POAHeader struct {
|
||||
Number *hexutil.Big `json:"number" gencodec:"required"`
|
||||
GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"`
|
||||
GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"`
|
||||
Time *hexutil.Big `json:"timestamp" gencodec:"required"`
|
||||
Time hexutil.Uint64 `json:"timestamp" gencodec:"required"`
|
||||
Extra hexutil.Bytes `json:"extraData" gencodec:"required"`
|
||||
Hash common.Hash `json:"hash"`
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
)
|
||||
|
||||
type Reader interface {
|
||||
@@ -48,7 +49,7 @@ func (ldbr *LevelDatabaseReader) GetBlockNumber(hash common.Hash) *uint64 {
|
||||
}
|
||||
|
||||
func (ldbr *LevelDatabaseReader) GetBlockReceipts(hash common.Hash, number uint64) types.Receipts {
|
||||
return rawdb.ReadReceipts(ldbr.reader, hash, number)
|
||||
return rawdb.ReadReceipts(ldbr.reader, hash, number, ¶ms.ChainConfig{})
|
||||
}
|
||||
|
||||
func (ldbr *LevelDatabaseReader) GetCanonicalHash(number uint64) common.Hash {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type AddressRepository struct{}
|
||||
|
||||
func (AddressRepository) GetOrCreateAddress(db *postgres.DB, address string) (int64, error) {
|
||||
stringAddressToCommonAddress := common.HexToAddress(address)
|
||||
hexAddress := stringAddressToCommonAddress.Hex()
|
||||
|
||||
var addressId int64
|
||||
getErr := db.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
|
||||
if getErr == sql.ErrNoRows {
|
||||
insertErr := db.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
|
||||
return addressId, insertErr
|
||||
}
|
||||
|
||||
return addressId, getErr
|
||||
}
|
||||
|
||||
func (AddressRepository) GetOrCreateAddressInTransaction(tx *sqlx.Tx, address string) (int64, error) {
|
||||
stringAddressToCommonAddress := common.HexToAddress(address)
|
||||
hexAddress := stringAddressToCommonAddress.Hex()
|
||||
|
||||
var addressId int64
|
||||
getErr := tx.Get(&addressId, `SELECT id FROM public.addresses WHERE address = $1`, hexAddress)
|
||||
if getErr == sql.ErrNoRows {
|
||||
insertErr := tx.QueryRow(`INSERT INTO public.addresses (address) VALUES($1) RETURNING id`, hexAddress).Scan(&addressId)
|
||||
return addressId, insertErr
|
||||
}
|
||||
|
||||
return addressId, getErr
|
||||
}
|
||||
|
||||
func (AddressRepository) GetAddressById(db *postgres.DB, id int64) (string, error) {
|
||||
var address string
|
||||
getErr := db.Get(&address, `SELECT address FROM public.addresses WHERE id = $1`, id)
|
||||
return address, getErr
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repositories_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/fakes"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("address lookup", func() {
|
||||
var (
|
||||
db *postgres.DB
|
||||
repo repositories.AddressRepository
|
||||
address = fakes.FakeAddress.Hex()
|
||||
)
|
||||
BeforeEach(func() {
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
repo = repositories.AddressRepository{}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
test_config.CleanTestDB(db)
|
||||
})
|
||||
|
||||
type dbAddress struct {
|
||||
Id int64
|
||||
Address string
|
||||
}
|
||||
|
||||
Describe("GetOrCreateAddress", func() {
|
||||
It("creates an address record", func() {
|
||||
addressId, createErr := repo.GetOrCreateAddress(db, address)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
var actualAddress dbAddress
|
||||
getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
expectedAddress := dbAddress{Id: addressId, Address: address}
|
||||
Expect(actualAddress).To(Equal(expectedAddress))
|
||||
})
|
||||
|
||||
It("returns the existing record id if the address already exists", func() {
|
||||
createId, createErr := repo.GetOrCreateAddress(db, address)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
getId, getErr := repo.GetOrCreateAddress(db, address)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
|
||||
var addressCount int
|
||||
addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`)
|
||||
Expect(addressErr).NotTo(HaveOccurred())
|
||||
Expect(addressCount).To(Equal(1))
|
||||
Expect(createId).To(Equal(getId))
|
||||
})
|
||||
|
||||
It("gets upper-cased addresses", func() {
|
||||
upperAddress := strings.ToUpper(address)
|
||||
upperAddressId, createErr := repo.GetOrCreateAddress(db, upperAddress)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
|
||||
})
|
||||
|
||||
It("gets lower-cased addresses", func() {
|
||||
lowerAddress := strings.ToLower(address)
|
||||
upperAddressId, createErr := repo.GetOrCreateAddress(db, lowerAddress)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
mixedCaseAddressId, getErr := repo.GetOrCreateAddress(db, address)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetOrCreateAddressInTransaction", func() {
|
||||
var (
|
||||
tx *sqlx.Tx
|
||||
txErr error
|
||||
)
|
||||
BeforeEach(func() {
|
||||
tx, txErr = db.Beginx()
|
||||
Expect(txErr).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
tx.Rollback()
|
||||
})
|
||||
|
||||
It("creates an address record", func() {
|
||||
addressId, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
commitErr := tx.Commit()
|
||||
Expect(commitErr).NotTo(HaveOccurred())
|
||||
|
||||
var actualAddress dbAddress
|
||||
getErr := db.Get(&actualAddress, `SELECT id, address FROM public.addresses LIMIT 1`)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
expectedAddress := dbAddress{Id: addressId, Address: address}
|
||||
Expect(actualAddress).To(Equal(expectedAddress))
|
||||
})
|
||||
|
||||
It("returns the existing record id if the address already exists", func() {
|
||||
_, createErr := repo.GetOrCreateAddressInTransaction(tx, address)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
_, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
tx.Commit()
|
||||
|
||||
var addressCount int
|
||||
addressErr := db.Get(&addressCount, `SELECT count(*) FROM public.addresses`)
|
||||
Expect(addressErr).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("gets upper-cased addresses", func() {
|
||||
upperAddress := strings.ToUpper(address)
|
||||
upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, upperAddress)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
tx.Commit()
|
||||
|
||||
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
|
||||
})
|
||||
|
||||
It("gets lower-cased addresses", func() {
|
||||
lowerAddress := strings.ToLower(address)
|
||||
upperAddressId, createErr := repo.GetOrCreateAddressInTransaction(tx, lowerAddress)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
mixedCaseAddressId, getErr := repo.GetOrCreateAddressInTransaction(tx, address)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
tx.Commit()
|
||||
|
||||
Expect(upperAddressId).To(Equal(mixedCaseAddressId))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetAddressById", func() {
|
||||
It("gets and address by it's id", func() {
|
||||
addressId, createErr := repo.GetOrCreateAddress(db, address)
|
||||
Expect(createErr).NotTo(HaveOccurred())
|
||||
|
||||
actualAddress, getErr := repo.GetAddressById(db, addressId)
|
||||
Expect(getErr).NotTo(HaveOccurred())
|
||||
Expect(actualAddress).To(Equal(address))
|
||||
})
|
||||
|
||||
It("returns an error if the id doesn't exist", func() {
|
||||
_, getErr := repo.GetAddressById(db, 0)
|
||||
Expect(getErr).To(HaveOccurred())
|
||||
Expect(getErr).To(MatchError("sql: no rows in result set"))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -234,7 +234,8 @@ func (blockRepository BlockRepository) createTransaction(tx *sqlx.Tx, blockId in
|
||||
return err
|
||||
}
|
||||
if hasReceipt(transaction) {
|
||||
receiptId, err := blockRepository.createReceipt(tx, blockId, transaction.Receipt)
|
||||
receiptRepo := FullSyncReceiptRepository{}
|
||||
receiptId, err := receiptRepo.CreateFullSyncReceiptInTx(blockId, transaction.Receipt, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -256,22 +257,6 @@ func hasReceipt(transaction core.TransactionModel) bool {
|
||||
return transaction.Receipt.TxHash != ""
|
||||
}
|
||||
|
||||
func (blockRepository BlockRepository) createReceipt(tx *sqlx.Tx, blockId int64, receipt core.Receipt) (int, error) {
|
||||
//Not currently persisting log bloom filters
|
||||
var receiptId int
|
||||
err := tx.QueryRow(
|
||||
`INSERT INTO full_sync_receipts
|
||||
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId)
|
||||
if err != nil {
|
||||
log.Error("createReceipt: error inserting receipt: ", err)
|
||||
return receiptId, err
|
||||
}
|
||||
return receiptId, nil
|
||||
}
|
||||
|
||||
func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, bool) {
|
||||
var retrievedBlockHash string
|
||||
// TODO: handle possible error
|
||||
@@ -283,7 +268,7 @@ func (blockRepository BlockRepository) getBlockHash(block core.Block) (string, b
|
||||
return retrievedBlockHash, blockExists(retrievedBlockHash)
|
||||
}
|
||||
|
||||
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int) error {
|
||||
func (blockRepository BlockRepository) createLogs(tx *sqlx.Tx, logs []core.Log, receiptId int64) error {
|
||||
for _, tlog := range logs {
|
||||
_, err := tx.Exec(
|
||||
`INSERT INTO logs (block_number, address, tx_hash, index, topic0, topic1, topic2, topic3, data, receipt_id)
|
||||
|
||||
@@ -80,7 +80,7 @@ var _ = Describe("Saving blocks", func() {
|
||||
blockNonce := "0x881db2ca900682e9a9"
|
||||
miner := "x123"
|
||||
extraData := "xextraData"
|
||||
blockTime := int64(1508981640)
|
||||
blockTime := uint64(1508981640)
|
||||
uncleHash := "x789"
|
||||
blockSize := string("1000")
|
||||
difficulty := int64(10)
|
||||
@@ -98,7 +98,7 @@ var _ = Describe("Saving blocks", func() {
|
||||
Number: blockNumber,
|
||||
ParentHash: blockParentHash,
|
||||
Size: blockSize,
|
||||
Time: blockTime,
|
||||
Time: uint64(blockTime),
|
||||
UncleHash: uncleHash,
|
||||
UnclesReward: unclesReward,
|
||||
}
|
||||
|
||||
+13
-25
@@ -26,17 +26,17 @@ import (
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
)
|
||||
|
||||
type ReceiptRepository struct {
|
||||
type FullSyncReceiptRepository struct {
|
||||
*postgres.DB
|
||||
}
|
||||
|
||||
func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
|
||||
func (receiptRepository FullSyncReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error {
|
||||
tx, err := receiptRepository.DB.Beginx()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, receipt := range receipts {
|
||||
receiptId, err := createReceipt(receipt, blockId, tx)
|
||||
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, receipt, tx)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
@@ -53,21 +53,6 @@ func (receiptRepository ReceiptRepository) CreateReceiptsAndLogs(blockId int64,
|
||||
return nil
|
||||
}
|
||||
|
||||
func createReceipt(receipt core.Receipt, blockId int64, tx *sqlx.Tx) (int64, error) {
|
||||
var receiptId int64
|
||||
err := tx.QueryRow(
|
||||
`INSERT INTO full_sync_receipts
|
||||
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId,
|
||||
).Scan(&receiptId)
|
||||
if err != nil {
|
||||
logrus.Error("createReceipt: Error inserting: ", err)
|
||||
}
|
||||
return receiptId, err
|
||||
}
|
||||
|
||||
func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
|
||||
for _, log := range logs {
|
||||
_, err := tx.Exec(
|
||||
@@ -83,27 +68,30 @@ func createLogs(logs []core.Log, receiptId int64, tx *sqlx.Tx) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (receiptRepository ReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
|
||||
tx, _ := receiptRepository.DB.Beginx()
|
||||
func (FullSyncReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
|
||||
var receiptId int64
|
||||
addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
|
||||
if getAddressErr != nil {
|
||||
logrus.Error("createReceipt: Error getting address id: ", getAddressErr)
|
||||
return receiptId, getAddressErr
|
||||
}
|
||||
err := tx.QueryRow(
|
||||
`INSERT INTO full_sync_receipts
|
||||
(contract_address, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
|
||||
(contract_address_id, tx_hash, cumulative_gas_used, gas_used, state_root, status, block_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
receipt.ContractAddress, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId)
|
||||
addressId, receipt.TxHash, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, blockId).Scan(&receiptId)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
logrus.Warning("CreateReceipt: error inserting receipt: ", err)
|
||||
return receiptId, err
|
||||
}
|
||||
tx.Commit()
|
||||
return receiptId, nil
|
||||
}
|
||||
|
||||
func (receiptRepository ReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) {
|
||||
func (receiptRepository FullSyncReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) {
|
||||
row := receiptRepository.DB.QueryRow(
|
||||
`SELECT contract_address,
|
||||
`SELECT contract_address_id,
|
||||
tx_hash,
|
||||
cumulative_gas_used,
|
||||
gas_used,
|
||||
+7
-7
@@ -30,7 +30,7 @@ import (
|
||||
var _ = Describe("Receipt Repository", func() {
|
||||
var blockRepository datastore.BlockRepository
|
||||
var logRepository datastore.LogRepository
|
||||
var receiptRepository datastore.ReceiptRepository
|
||||
var receiptRepository datastore.FullSyncReceiptRepository
|
||||
var db *postgres.DB
|
||||
var node core.Node
|
||||
BeforeEach(func() {
|
||||
@@ -44,7 +44,7 @@ var _ = Describe("Receipt Repository", func() {
|
||||
test_config.CleanTestDB(db)
|
||||
blockRepository = repositories.NewBlockRepository(db)
|
||||
logRepository = repositories.LogRepository{DB: db}
|
||||
receiptRepository = repositories.ReceiptRepository{DB: db}
|
||||
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
|
||||
})
|
||||
|
||||
Describe("Saving multiple receipts", func() {
|
||||
@@ -84,12 +84,12 @@ var _ = Describe("Receipt Repository", func() {
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
persistedReceiptOne, err := receiptRepository.GetReceipt(txHashOne)
|
||||
persistedReceiptOne, err := receiptRepository.GetFullSyncReceipt(txHashOne)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(persistedReceiptOne).NotTo(BeNil())
|
||||
Expect(persistedReceiptOne.TxHash).To(Equal(txHashOne))
|
||||
|
||||
persistedReceiptTwo, err := receiptRepository.GetReceipt(txHashTwo)
|
||||
persistedReceiptTwo, err := receiptRepository.GetFullSyncReceipt(txHashTwo)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(persistedReceiptTwo).NotTo(BeNil())
|
||||
Expect(persistedReceiptTwo.TxHash).To(Equal(txHashTwo))
|
||||
@@ -124,7 +124,7 @@ var _ = Describe("Receipt Repository", func() {
|
||||
_, err := blockRepository.CreateOrUpdateBlock(block)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
receipt, err := receiptRepository.GetReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
|
||||
receipt, err := receiptRepository.GetFullSyncReceipt("0xe340558980f89d5f86045ac11e5cc34e4bcec20f9f1e2a427aa39d87114e8223")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
//Not currently serializing bloom logs
|
||||
Expect(receipt.Bloom).To(Equal(core.Receipt{}.Bloom))
|
||||
@@ -136,7 +136,7 @@ var _ = Describe("Receipt Repository", func() {
|
||||
})
|
||||
|
||||
It("returns ErrReceiptDoesNotExist when receipt does not exist", func() {
|
||||
receipt, err := receiptRepository.GetReceipt("DOES NOT EXIST")
|
||||
receipt, err := receiptRepository.GetFullSyncReceipt("DOES NOT EXIST")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(receipt).To(BeZero())
|
||||
})
|
||||
@@ -154,7 +154,7 @@ var _ = Describe("Receipt Repository", func() {
|
||||
_, err := blockRepository.CreateOrUpdateBlock(block)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = receiptRepository.GetReceipt(receipt.TxHash)
|
||||
_, err = receiptRepository.GetFullSyncReceipt(receipt.TxHash)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
})
|
||||
})
|
||||
@@ -85,22 +85,6 @@ func (repository HeaderRepository) CreateTransactionInTx(tx *sqlx.Tx, headerID i
|
||||
return txId, err
|
||||
}
|
||||
|
||||
func (repository HeaderRepository) CreateReceiptInTx(tx *sqlx.Tx, headerID, transactionID int64, receipt core.Receipt) (int64, error) {
|
||||
var receiptId int64
|
||||
err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
|
||||
(header_id, transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (header_id, transaction_id) DO UPDATE
|
||||
SET (contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9)
|
||||
RETURNING id`,
|
||||
headerID, transactionID, receipt.ContractAddress, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId)
|
||||
if err != nil {
|
||||
log.Error("header_repository: error inserting receipt: ", err)
|
||||
return receiptId, err
|
||||
}
|
||||
return receiptId, err
|
||||
}
|
||||
|
||||
func (repository HeaderRepository) GetHeader(blockNumber int64) (core.Header, error) {
|
||||
var header core.Header
|
||||
err := repository.database.Get(&header, `SELECT id, block_number, hash, raw, block_timestamp FROM headers WHERE block_number = $1 AND eth_node_fingerprint = $2`,
|
||||
|
||||
@@ -216,26 +216,40 @@ var _ = Describe("Block header repository", func() {
|
||||
Rlp: []byte{1, 2, 3},
|
||||
}
|
||||
|
||||
_, receiptErr := repo.CreateReceiptInTx(tx, headerID, txId, receipt)
|
||||
receiptRepo := repositories.HeaderSyncReceiptRepository{}
|
||||
_, receiptErr := receiptRepo.CreateHeaderSyncReceiptInTx(headerID, txId, receipt, tx)
|
||||
Expect(receiptErr).ToNot(HaveOccurred())
|
||||
commitErr := tx.Commit()
|
||||
Expect(commitErr).ToNot(HaveOccurred())
|
||||
Expect(receiptErr).ToNot(HaveOccurred())
|
||||
|
||||
type idModel struct {
|
||||
TransactionId int64 `db:"transaction_id"`
|
||||
core.Receipt
|
||||
TransactionId int64 `db:"transaction_id"`
|
||||
ContractAddressId int64 `db:"contract_address_id"`
|
||||
CumulativeGasUsed uint64 `db:"cumulative_gas_used"`
|
||||
GasUsed uint64 `db:"gas_used"`
|
||||
StateRoot string `db:"state_root"`
|
||||
Status int
|
||||
TxHash string `db:"tx_hash"`
|
||||
Rlp []byte `db:"rlp"`
|
||||
}
|
||||
|
||||
var addressId int64
|
||||
getAddressErr := db.Get(&addressId, `SELECT id FROM addresses WHERE address = $1`, contractAddr.Hex())
|
||||
Expect(getAddressErr).NotTo(HaveOccurred())
|
||||
|
||||
var dbReceipt idModel
|
||||
err = db.Get(&dbReceipt,
|
||||
`SELECT transaction_id, contract_address, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
|
||||
getReceiptErr := db.Get(&dbReceipt,
|
||||
`SELECT transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
|
||||
FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(getReceiptErr).NotTo(HaveOccurred())
|
||||
|
||||
Expect(dbReceipt.TransactionId).To(Equal(txId))
|
||||
Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
|
||||
Expect(dbReceipt.ContractAddress).To(Equal(contractAddr.Hex()))
|
||||
Expect(dbReceipt.ContractAddressId).To(Equal(addressId))
|
||||
Expect(dbReceipt.CumulativeGasUsed).To(Equal(uint64(100)))
|
||||
Expect(dbReceipt.GasUsed).To(Equal(uint64(10)))
|
||||
Expect(dbReceipt.StateRoot).To(Equal(stateRoot.Hex()))
|
||||
Expect(dbReceipt.Status).To(Equal(0))
|
||||
Expect(dbReceipt.Rlp).To(Equal([]byte{1, 2, 3}))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
)
|
||||
|
||||
type HeaderSyncReceiptRepository struct{}
|
||||
|
||||
func (HeaderSyncReceiptRepository) CreateHeaderSyncReceiptInTx(headerID, transactionID int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
|
||||
var receiptId int64
|
||||
addressId, getAddressErr := AddressRepository{}.GetOrCreateAddressInTransaction(tx, receipt.ContractAddress)
|
||||
if getAddressErr != nil {
|
||||
log.Error("createReceipt: Error getting address id: ", getAddressErr)
|
||||
return receiptId, getAddressErr
|
||||
}
|
||||
err := tx.QueryRowx(`INSERT INTO public.header_sync_receipts
|
||||
(header_id, transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (header_id, transaction_id) DO UPDATE
|
||||
SET (contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp) = ($3, $4::NUMERIC, $5::NUMERIC, $6, $7, $8, $9)
|
||||
RETURNING id`,
|
||||
headerID, transactionID, addressId, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.StateRoot, receipt.Status, receipt.TxHash, receipt.Rlp).Scan(&receiptId)
|
||||
if err != nil {
|
||||
log.Error("header_repository: error inserting receipt: ", err)
|
||||
return receiptId, err
|
||||
}
|
||||
return receiptId, err
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package repositories_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
|
||||
"github.com/vulcanize/vulcanizedb/test_config"
|
||||
)
|
||||
|
||||
var _ = Describe("Header Sync Receipt Repo", func() {
|
||||
var (
|
||||
rawHeader []byte
|
||||
err error
|
||||
timestamp string
|
||||
db *postgres.DB
|
||||
receiptRepo repositories.HeaderSyncReceiptRepository
|
||||
headerRepo repositories.HeaderRepository
|
||||
header core.Header
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rawHeader, err = json.Marshal(types.Header{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
timestamp = big.NewInt(123456789).String()
|
||||
|
||||
db = test_config.NewTestDB(test_config.NewTestNode())
|
||||
test_config.CleanTestDB(db)
|
||||
receiptRepo = repositories.HeaderSyncReceiptRepository{}
|
||||
headerRepo = repositories.NewHeaderRepository(db)
|
||||
header = core.Header{
|
||||
BlockNumber: 100,
|
||||
Hash: common.BytesToHash([]byte{1, 2, 3, 4, 5}).Hex(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
})
|
||||
Describe("creating a receipt", func() {
|
||||
It("adds a receipt in a tx", func() {
|
||||
headerID, err := headerRepo.CreateOrUpdateHeader(header)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
fromAddress := common.HexToAddress("0x1234")
|
||||
toAddress := common.HexToAddress("0x5678")
|
||||
txHash := common.HexToHash("0x9876")
|
||||
txIndex := big.NewInt(123)
|
||||
transaction := core.TransactionModel{
|
||||
Data: []byte{},
|
||||
From: fromAddress.Hex(),
|
||||
GasLimit: 0,
|
||||
GasPrice: 0,
|
||||
Hash: txHash.Hex(),
|
||||
Nonce: 0,
|
||||
Raw: []byte{},
|
||||
To: toAddress.Hex(),
|
||||
TxIndex: txIndex.Int64(),
|
||||
Value: "0",
|
||||
}
|
||||
tx, err := db.Beginx()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
txId, txErr := headerRepo.CreateTransactionInTx(tx, headerID, transaction)
|
||||
Expect(txErr).ToNot(HaveOccurred())
|
||||
|
||||
contractAddr := common.HexToAddress("0x1234")
|
||||
stateRoot := common.HexToHash("0x5678")
|
||||
receipt := core.Receipt{
|
||||
ContractAddress: contractAddr.Hex(),
|
||||
TxHash: txHash.Hex(),
|
||||
GasUsed: 10,
|
||||
CumulativeGasUsed: 100,
|
||||
StateRoot: stateRoot.Hex(),
|
||||
Rlp: []byte{1, 2, 3},
|
||||
}
|
||||
|
||||
_, receiptErr := receiptRepo.CreateHeaderSyncReceiptInTx(headerID, txId, receipt, tx)
|
||||
Expect(receiptErr).ToNot(HaveOccurred())
|
||||
commitErr := tx.Commit()
|
||||
Expect(commitErr).ToNot(HaveOccurred())
|
||||
|
||||
type idModel struct {
|
||||
TransactionId int64 `db:"transaction_id"`
|
||||
ContractAddressId int64 `db:"contract_address_id"`
|
||||
CumulativeGasUsed uint64 `db:"cumulative_gas_used"`
|
||||
GasUsed uint64 `db:"gas_used"`
|
||||
StateRoot string `db:"state_root"`
|
||||
Status int
|
||||
TxHash string `db:"tx_hash"`
|
||||
Rlp []byte `db:"rlp"`
|
||||
}
|
||||
|
||||
var addressId int64
|
||||
getAddressErr := db.Get(&addressId, `SELECT id FROM addresses WHERE address = $1`, contractAddr.Hex())
|
||||
Expect(getAddressErr).NotTo(HaveOccurred())
|
||||
|
||||
var dbReceipt idModel
|
||||
getReceiptErr := db.Get(&dbReceipt,
|
||||
`SELECT transaction_id, contract_address_id, cumulative_gas_used, gas_used, state_root, status, tx_hash, rlp
|
||||
FROM public.header_sync_receipts WHERE header_id = $1`, headerID)
|
||||
Expect(getReceiptErr).NotTo(HaveOccurred())
|
||||
|
||||
Expect(dbReceipt.TransactionId).To(Equal(txId))
|
||||
Expect(dbReceipt.TxHash).To(Equal(txHash.Hex()))
|
||||
Expect(dbReceipt.ContractAddressId).To(Equal(addressId))
|
||||
Expect(dbReceipt.CumulativeGasUsed).To(Equal(uint64(100)))
|
||||
Expect(dbReceipt.GasUsed).To(Equal(uint64(10)))
|
||||
Expect(dbReceipt.StateRoot).To(Equal(stateRoot.Hex()))
|
||||
Expect(dbReceipt.Status).To(Equal(0))
|
||||
Expect(dbReceipt.Rlp).To(Equal([]byte{1, 2, 3}))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -34,7 +34,7 @@ var _ = Describe("Logs Repository", func() {
|
||||
var db *postgres.DB
|
||||
var blockRepository datastore.BlockRepository
|
||||
var logsRepository datastore.LogRepository
|
||||
var receiptRepository datastore.ReceiptRepository
|
||||
var receiptRepository datastore.FullSyncReceiptRepository
|
||||
var node core.Node
|
||||
|
||||
BeforeEach(func() {
|
||||
@@ -48,14 +48,16 @@ var _ = Describe("Logs Repository", func() {
|
||||
test_config.CleanTestDB(db)
|
||||
blockRepository = repositories.NewBlockRepository(db)
|
||||
logsRepository = repositories.LogRepository{DB: db}
|
||||
receiptRepository = repositories.ReceiptRepository{DB: db}
|
||||
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
|
||||
})
|
||||
|
||||
It("returns the log when it exists", func() {
|
||||
blockNumber := int64(12345)
|
||||
blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
|
||||
tx, _ := db.Beginx()
|
||||
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
|
||||
tx.Commit()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = logsRepository.CreateLogs([]core.Log{{
|
||||
BlockNumber: blockNumber,
|
||||
@@ -91,7 +93,9 @@ var _ = Describe("Logs Repository", func() {
|
||||
blockNumber := int64(12345)
|
||||
blockId, err := blockRepository.CreateOrUpdateBlock(core.Block{Number: blockNumber})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
|
||||
tx, _ := db.Beginx()
|
||||
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
|
||||
tx.Commit()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = logsRepository.CreateLogs([]core.Log{{
|
||||
|
||||
@@ -19,6 +19,7 @@ package repositories_test
|
||||
import (
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
|
||||
@@ -32,7 +33,7 @@ var _ = Describe("Watched Events Repository", func() {
|
||||
var blocksRepository datastore.BlockRepository
|
||||
var filterRepository datastore.FilterRepository
|
||||
var logRepository datastore.LogRepository
|
||||
var receiptRepository datastore.ReceiptRepository
|
||||
var receiptRepository datastore.FullSyncReceiptRepository
|
||||
var watchedEventRepository datastore.WatchedEventRepository
|
||||
|
||||
BeforeEach(func() {
|
||||
@@ -41,7 +42,7 @@ var _ = Describe("Watched Events Repository", func() {
|
||||
blocksRepository = repositories.NewBlockRepository(db)
|
||||
filterRepository = repositories.FilterRepository{DB: db}
|
||||
logRepository = repositories.LogRepository{DB: db}
|
||||
receiptRepository = repositories.ReceiptRepository{DB: db}
|
||||
receiptRepository = repositories.FullSyncReceiptRepository{DB: db}
|
||||
watchedEventRepository = repositories.WatchedEventRepository{DB: db}
|
||||
})
|
||||
|
||||
@@ -79,7 +80,10 @@ var _ = Describe("Watched Events Repository", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{})
|
||||
tx, txBeginErr := db.Beginx()
|
||||
Expect(txBeginErr).NotTo(HaveOccurred())
|
||||
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
|
||||
tx.Commit()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = logRepository.CreateLogs(logs, receiptId)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
@@ -136,7 +140,9 @@ var _ = Describe("Watched Events Repository", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
blockId, err := blocksRepository.CreateOrUpdateBlock(core.Block{Hash: "Ox123"})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
receiptId, err := receiptRepository.CreateReceipt(blockId, core.Receipt{TxHash: "0x123"})
|
||||
tx, _ := db.Beginx()
|
||||
receiptId, err := receiptRepository.CreateFullSyncReceiptInTx(blockId, core.Receipt{}, tx)
|
||||
tx.Commit()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = logRepository.CreateLogs(logs, receiptId)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
@@ -17,10 +17,15 @@
|
||||
package datastore
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/filters"
|
||||
)
|
||||
|
||||
type AddressRepository interface {
|
||||
GetOrCreateAddress(address string) (int, error)
|
||||
}
|
||||
|
||||
type BlockRepository interface {
|
||||
CreateOrUpdateBlock(block core.Block) (int64, error)
|
||||
GetBlock(blockNumber int64) (core.Block, error)
|
||||
@@ -51,10 +56,14 @@ type LogRepository interface {
|
||||
GetLogs(address string, blockNumber int64) ([]core.Log, error)
|
||||
}
|
||||
|
||||
type ReceiptRepository interface {
|
||||
type FullSyncReceiptRepository interface {
|
||||
CreateReceiptsAndLogs(blockId int64, receipts []core.Receipt) error
|
||||
CreateReceipt(blockId int64, receipt core.Receipt) (int64, error)
|
||||
GetReceipt(txHash string) (core.Receipt, error)
|
||||
CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error)
|
||||
GetFullSyncReceipt(txHash string) (core.Receipt, error)
|
||||
}
|
||||
|
||||
type HeaderSyncReceiptRepository interface {
|
||||
CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error)
|
||||
}
|
||||
|
||||
type WatchedEventRepository interface {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package fakes
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
@@ -49,11 +50,11 @@ func (mrr *MockReceiptRepository) CreateReceiptsAndLogs(blockId int64, receipts
|
||||
return mrr.createReceiptsAndLogsReturnErr
|
||||
}
|
||||
|
||||
func (mrr *MockReceiptRepository) CreateReceipt(blockId int64, receipt core.Receipt) (int64, error) {
|
||||
func (mrr *MockReceiptRepository) CreateFullSyncReceiptInTx(blockId int64, receipt core.Receipt, tx *sqlx.Tx) (int64, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (mrr *MockReceiptRepository) GetReceipt(txHash string) (core.Receipt, error) {
|
||||
func (mrr *MockReceiptRepository) GetFullSyncReceipt(txHash string) (core.Receipt, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
|
||||
@@ -129,6 +129,9 @@ func (blockChain *BlockChain) GetTransactions(transactionHashes []common.Hash) (
|
||||
|
||||
func (blockChain *BlockChain) LastBlock() (*big.Int, error) {
|
||||
block, err := blockChain.ethClient.HeaderByNumber(context.Background(), nil)
|
||||
if err != nil {
|
||||
return big.NewInt(0), err
|
||||
}
|
||||
return block.Number, err
|
||||
}
|
||||
|
||||
@@ -159,7 +162,7 @@ func (blockChain *BlockChain) getPOAHeader(blockNumber int64) (header core.Heade
|
||||
Number: POAHeader.Number.ToInt(),
|
||||
GasLimit: uint64(POAHeader.GasLimit),
|
||||
GasUsed: uint64(POAHeader.GasUsed),
|
||||
Time: POAHeader.Time.ToInt(),
|
||||
Time: uint64(POAHeader.Time),
|
||||
Extra: POAHeader.Extra,
|
||||
}, POAHeader.Hash.String()), nil
|
||||
}
|
||||
@@ -208,7 +211,7 @@ func (blockChain *BlockChain) getPOAHeaders(blockNumbers []int64) (headers []cor
|
||||
Number: POAHeader.Number.ToInt(),
|
||||
GasLimit: uint64(POAHeader.GasLimit),
|
||||
GasUsed: uint64(POAHeader.GasUsed),
|
||||
Time: POAHeader.Time.ToInt(),
|
||||
Time: uint64(POAHeader.Time),
|
||||
Extra: POAHeader.Extra,
|
||||
}, POAHeader.Hash.String())
|
||||
|
||||
|
||||
@@ -26,10 +26,10 @@ type ColdImporter struct {
|
||||
blockRepository datastore.BlockRepository
|
||||
converter common.BlockConverter
|
||||
ethDB ethereum.Database
|
||||
receiptRepository datastore.ReceiptRepository
|
||||
receiptRepository datastore.FullSyncReceiptRepository
|
||||
}
|
||||
|
||||
func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.ReceiptRepository, converter common.BlockConverter) *ColdImporter {
|
||||
func NewColdImporter(ethDB ethereum.Database, blockRepository datastore.BlockRepository, receiptRepository datastore.FullSyncReceiptRepository, converter common.BlockConverter) *ColdImporter {
|
||||
return &ColdImporter{
|
||||
blockRepository: blockRepository,
|
||||
converter: converter,
|
||||
|
||||
@@ -19,6 +19,7 @@ package common
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
@@ -52,7 +53,7 @@ func (bc BlockConverter) ToCoreBlock(gethBlock *types.Block) (core.Block, error)
|
||||
Number: gethBlock.Number().Int64(),
|
||||
ParentHash: gethBlock.ParentHash().Hex(),
|
||||
Size: gethBlock.Size().String(),
|
||||
Time: gethBlock.Time().Int64(),
|
||||
Time: gethBlock.Time(),
|
||||
Transactions: transactions,
|
||||
UncleHash: gethBlock.UncleHash().Hex(),
|
||||
}
|
||||
@@ -81,7 +82,7 @@ func (bc BlockConverter) ToCoreUncle(block core.Block, uncles []*types.Header) (
|
||||
Hash: uncle.Hash().Hex(),
|
||||
Raw: raw,
|
||||
Reward: thisUncleReward.String(),
|
||||
Timestamp: uncle.Time.String(),
|
||||
Timestamp: strconv.FormatUint(uncle.Time, 10),
|
||||
}
|
||||
coreUncles = append(coreUncles, coreUncle)
|
||||
totalUncleRewards.Add(totalUncleRewards, thisUncleReward)
|
||||
|
||||
@@ -44,7 +44,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
|
||||
extraData, _ := hexutil.Decode("0xe4b883e5bda9e7a59ee4bb99e9b1bc")
|
||||
nonce := types.BlockNonce{10}
|
||||
number := int64(1)
|
||||
time := int64(140000000)
|
||||
time := uint64(140000000)
|
||||
|
||||
header := types.Header{
|
||||
Difficulty: difficulty,
|
||||
@@ -55,7 +55,7 @@ var _ = Describe("Conversion of GethBlock to core.Block", func() {
|
||||
Nonce: nonce,
|
||||
Number: big.NewInt(number),
|
||||
ParentHash: common.Hash{64},
|
||||
Time: big.NewInt(time),
|
||||
Time: time,
|
||||
UncleHash: common.Hash{128},
|
||||
}
|
||||
block := types.NewBlock(&header, []*types.Transaction{}, []*types.Header{}, []*types.Receipt{})
|
||||
|
||||
@@ -18,6 +18,7 @@ package common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
@@ -34,7 +35,7 @@ func (converter HeaderConverter) Convert(gethHeader *types.Header, blockHash str
|
||||
Hash: blockHash,
|
||||
BlockNumber: gethHeader.Number.Int64(),
|
||||
Raw: rawHeader,
|
||||
Timestamp: gethHeader.Time.String(),
|
||||
Timestamp: strconv.FormatUint(gethHeader.Time, 10),
|
||||
}
|
||||
return coreHeader
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ package common_test
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"strconv"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
@@ -37,7 +38,7 @@ var _ = Describe("Block header converter", func() {
|
||||
ParentHash: common.HexToHash("0xParent"),
|
||||
ReceiptHash: common.HexToHash("0xReceipt"),
|
||||
Root: common.HexToHash("0xRoot"),
|
||||
Time: big.NewInt(123456789),
|
||||
Time: uint64(123456789),
|
||||
TxHash: common.HexToHash("0xTransaction"),
|
||||
UncleHash: common.HexToHash("0xUncle"),
|
||||
}
|
||||
@@ -48,7 +49,7 @@ var _ = Describe("Block header converter", func() {
|
||||
|
||||
Expect(coreHeader.BlockNumber).To(Equal(gethHeader.Number.Int64()))
|
||||
Expect(coreHeader.Hash).To(Equal(hash))
|
||||
Expect(coreHeader.Timestamp).To(Equal(gethHeader.Time.String()))
|
||||
Expect(coreHeader.Timestamp).To(Equal(strconv.FormatUint(gethHeader.Time, 10)))
|
||||
})
|
||||
|
||||
It("includes raw bytes for header as JSON", func() {
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rpc_test
|
||||
|
||||
import (
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/vulcanizedb/pkg/core"
|
||||
@@ -35,7 +36,7 @@ func PopulateMissingBlocks(blockchain core.BlockChain, blockRepository datastore
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("Backfilling %d blocks\n\n", len(blockRange))
|
||||
log.Debug(getBlockRangeString(blockRange))
|
||||
_, err = RetrieveAndUpdateBlocks(blockchain, blockRepository, blockRange)
|
||||
if err != nil {
|
||||
log.Error("PopulateMissingBlocks: error gettings/updating blocks: ", err)
|
||||
@@ -61,3 +62,7 @@ func RetrieveAndUpdateBlocks(blockchain core.BlockChain, blockRepository datasto
|
||||
}
|
||||
return len(blockNumbers), nil
|
||||
}
|
||||
|
||||
func getBlockRangeString(blockRange []int64) string {
|
||||
return fmt.Sprintf("Backfilling |%v| blocks", len(blockRange))
|
||||
}
|
||||
|
||||
@@ -39,10 +39,10 @@ func PopulateMissingHeaders(blockChain core.BlockChain, headerRepository datasto
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("Backfilling %d blocks\n\n", len(blockNumbers))
|
||||
log.Debug(getBlockRangeString(blockNumbers))
|
||||
_, err = RetrieveAndUpdateHeaders(blockChain, headerRepository, blockNumbers)
|
||||
if err != nil {
|
||||
log.Error("PopulateMissingHeaders: Error getting/updating headers:", err)
|
||||
log.Error("PopulateMissingHeaders: Error getting/updating headers: ", err)
|
||||
return 0, err
|
||||
}
|
||||
return len(blockNumbers), nil
|
||||
|
||||
@@ -50,6 +50,6 @@ func MakeRange(min, max int64) []int64 {
|
||||
}
|
||||
|
||||
func (window ValidationWindow) GetString() string {
|
||||
return fmt.Sprintf("Validating Blocks |%v|-- Validation Window --|%v}|",
|
||||
return fmt.Sprintf("Validating Blocks |%v|-- Validation Window --|%v|",
|
||||
window.LowerBound, window.UpperBound)
|
||||
}
|
||||
|
||||
@@ -17,13 +17,15 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/lib/pq"
|
||||
"github.com/pressly/goose"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/config"
|
||||
"github.com/vulcanize/vulcanizedb/pkg/plugin/helpers"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
@@ -36,6 +38,7 @@ type manager struct {
|
||||
GenConfig config.Plugin
|
||||
DBConfig config.Database
|
||||
tmpMigDir string
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// Manager requires both filled in generator and database configs
|
||||
@@ -46,6 +49,22 @@ func NewMigrationManager(gc config.Plugin, dbc config.Database) *manager {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *manager) setDB() error {
|
||||
var pgStr string
|
||||
if len(m.DBConfig.User) > 0 && len(m.DBConfig.Password) > 0 {
|
||||
pgStr = fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
|
||||
m.DBConfig.User, m.DBConfig.Password, m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
|
||||
} else {
|
||||
pgStr = fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
|
||||
}
|
||||
dbConnector, err := pq.NewConnector(pgStr)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("can't connect to db: %s", err.Error()))
|
||||
}
|
||||
m.db = sql.OpenDB(dbConnector)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *manager) RunMigrations() error {
|
||||
// Get paths to db migrations from the plugin config
|
||||
paths, err := m.GenConfig.GetMigrationsPaths()
|
||||
@@ -73,17 +92,19 @@ func (m *manager) RunMigrations() error {
|
||||
// Setup a temporary directory to hold transformer db migrations
|
||||
func (m *manager) setupMigrationEnv() error {
|
||||
var err error
|
||||
m.tmpMigDir, err = helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home+".plugin_migrations"))
|
||||
m.tmpMigDir, err = helpers.CleanPath(filepath.Join("$GOPATH/src", m.GenConfig.Home, ".plugin_migrations"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.RemoveAll(m.tmpMigDir)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("unable to remove file found at %s where tmp directory needs to be written", m.tmpMigDir))
|
||||
removeErr := os.RemoveAll(m.tmpMigDir)
|
||||
if removeErr != nil {
|
||||
removeErrString := "unable to remove file found at %s where tmp directory needs to be written: %s"
|
||||
return errors.New(fmt.Sprintf(removeErrString, m.tmpMigDir, removeErr.Error()))
|
||||
}
|
||||
err = os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("unable to create temporary migration directory %s", m.tmpMigDir))
|
||||
mkdirErr := os.Mkdir(m.tmpMigDir, os.FileMode(os.ModePerm))
|
||||
if mkdirErr != nil {
|
||||
mkdirErrString := "unable to create temporary migration directory %s: %s"
|
||||
return errors.New(fmt.Sprintf(mkdirErrString, m.tmpMigDir, mkdirErr.Error()))
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -120,28 +141,23 @@ func (m *manager) createMigrationCopies(paths []string) error {
|
||||
}
|
||||
|
||||
func (m *manager) fixAndRun(path string) error {
|
||||
// Setup DB if not set
|
||||
if m.db == nil {
|
||||
setErr := m.setDB()
|
||||
if setErr != nil {
|
||||
return errors.New(fmt.Sprintf("could not open db: %s", setErr.Error()))
|
||||
}
|
||||
}
|
||||
// Fix the migrations
|
||||
cmd := exec.Command("goose", "fix")
|
||||
cmd.Dir = m.tmpMigDir
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, err.Error()))
|
||||
fixErr := goose.Fix(m.tmpMigDir)
|
||||
if fixErr != nil {
|
||||
return errors.New(fmt.Sprintf("version fixing for plugin migrations at %s failed: %s", path, fixErr.Error()))
|
||||
}
|
||||
// Run the copied migrations with goose
|
||||
var pgStr string
|
||||
if len(m.DBConfig.User) > 0 && len(m.DBConfig.Password) > 0 {
|
||||
pgStr = fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
|
||||
m.DBConfig.User, m.DBConfig.Password, m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
|
||||
} else {
|
||||
pgStr = fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable", m.DBConfig.Hostname, m.DBConfig.Port, m.DBConfig.Name)
|
||||
upErr := goose.Up(m.db, m.tmpMigDir)
|
||||
if upErr != nil {
|
||||
return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, upErr.Error()))
|
||||
}
|
||||
cmd = exec.Command("goose", "postgres", pgStr, "up")
|
||||
cmd.Dir = m.tmpMigDir
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("db migrations for plugin transformers at %s failed: %s", path, err.Error()))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -105,6 +105,7 @@ func NewTestDB(node core.Node) *postgres.DB {
|
||||
}
|
||||
|
||||
func CleanTestDB(db *postgres.DB) {
|
||||
db.MustExec("DELETE FROM addresses")
|
||||
db.MustExec("DELETE FROM blocks")
|
||||
db.MustExec("DELETE FROM checked_headers")
|
||||
// can't delete from eth_nodes since this function is called after the required eth_node is persisted
|
||||
|
||||
+2
-2
@@ -31,7 +31,7 @@ import (
|
||||
func LoadPostgres(database config.Database, node core.Node) postgres.DB {
|
||||
db, err := postgres.NewDB(database, node)
|
||||
if err != nil {
|
||||
log.Fatalf("Error loading postgres\n%v", err)
|
||||
log.Fatal("Error loading postgres: ", err)
|
||||
}
|
||||
return *db
|
||||
}
|
||||
@@ -65,7 +65,7 @@ func GetAbi(abiFilepath string, contractHash string, network string) string {
|
||||
}
|
||||
_, err := geth.ParseAbi(contractAbiString)
|
||||
if err != nil {
|
||||
log.Fatalln("Invalid ABI")
|
||||
log.Fatalln("Invalid ABI: ", err)
|
||||
}
|
||||
return contractAbiString
|
||||
}
|
||||
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
.idea
|
||||
.DS_Store
|
||||
/server/server.exe
|
||||
/server/server
|
||||
CHANGELOG.md
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.x
|
||||
- tip
|
||||
|
||||
matrix:
|
||||
allow_failures:
|
||||
- go: tip
|
||||
fast_finish: true
|
||||
|
||||
before_install:
|
||||
- go get github.com/modocache/gover
|
||||
- go get github.com/mattn/goveralls
|
||||
- go get golang.org/x/tools/cmd/cover
|
||||
- go get golang.org/x/tools/cmd/goimports
|
||||
- go get github.com/golang/lint/golint
|
||||
- go get github.com/stretchr/testify/assert
|
||||
- go get github.com/gordonklaus/ineffassign
|
||||
|
||||
script:
|
||||
- gofiles=$(find ./ -name '*.go') && [ -z "$gofiles" ] || unformatted=$(goimports -l $gofiles) && [ -z "$unformatted" ] || (echo >&2 "Go files must be formatted with gofmt. Following files has problem:\n $unformatted" && false)
|
||||
- diff <(echo -n) <(gofmt -s -d .)
|
||||
- golint ./... # This won't break the build, just show warnings
|
||||
- ineffassign .
|
||||
- go vet ./...
|
||||
- go test -race -count=1 -coverprofile=queue.coverprofile ./queue
|
||||
- go test -race -count=1 -coverprofile=server.coverprofile ./server
|
||||
- go test -race -count=1 -coverprofile=main.coverprofile
|
||||
- $HOME/gopath/bin/gover
|
||||
- $HOME/gopath/bin/goveralls -coverprofile=gover.coverprofile -service travis-ci
|
||||
-201
@@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-145
@@ -1,145 +0,0 @@
|
||||
# BigCache [](https://travis-ci.org/allegro/bigcache) [](https://coveralls.io/github/allegro/bigcache?branch=master) [](https://godoc.org/github.com/allegro/bigcache) [](https://goreportcard.com/report/github.com/allegro/bigcache)
|
||||
|
||||
Fast, concurrent, evicting in-memory cache written to keep big number of entries without impact on performance.
|
||||
BigCache keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place,
|
||||
therefore entries (de)serialization in front of the cache will be needed in most use cases.
|
||||
|
||||
## Usage
|
||||
|
||||
### Simple initialization
|
||||
|
||||
```go
|
||||
import "github.com/allegro/bigcache"
|
||||
|
||||
cache, _ := bigcache.NewBigCache(bigcache.DefaultConfig(10 * time.Minute))
|
||||
|
||||
cache.Set("my-unique-key", []byte("value"))
|
||||
|
||||
entry, _ := cache.Get("my-unique-key")
|
||||
fmt.Println(string(entry))
|
||||
```
|
||||
|
||||
### Custom initialization
|
||||
|
||||
When cache load can be predicted in advance then it is better to use custom initialization because additional memory
|
||||
allocation can be avoided in that way.
|
||||
|
||||
```go
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/allegro/bigcache"
|
||||
)
|
||||
|
||||
config := bigcache.Config {
|
||||
// number of shards (must be a power of 2)
|
||||
Shards: 1024,
|
||||
// time after which entry can be evicted
|
||||
LifeWindow: 10 * time.Minute,
|
||||
// rps * lifeWindow, used only in initial memory allocation
|
||||
MaxEntriesInWindow: 1000 * 10 * 60,
|
||||
// max entry size in bytes, used only in initial memory allocation
|
||||
MaxEntrySize: 500,
|
||||
// prints information about additional memory allocation
|
||||
Verbose: true,
|
||||
// cache will not allocate more memory than this limit, value in MB
|
||||
// if value is reached then the oldest entries can be overridden for the new ones
|
||||
// 0 value means no size limit
|
||||
HardMaxCacheSize: 8192,
|
||||
// callback fired when the oldest entry is removed because of its
|
||||
// expiration time or no space left for the new entry. Default value is nil which
|
||||
// means no callback and it prevents from unwrapping the oldest entry.
|
||||
OnRemove: nil,
|
||||
}
|
||||
|
||||
cache, initErr := bigcache.NewBigCache(config)
|
||||
if initErr != nil {
|
||||
log.Fatal(initErr)
|
||||
}
|
||||
|
||||
cache.Set("my-unique-key", []byte("value"))
|
||||
|
||||
if entry, err := cache.Get("my-unique-key"); err == nil {
|
||||
fmt.Println(string(entry))
|
||||
}
|
||||
```
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Three caches were compared: bigcache, [freecache](https://github.com/coocood/freecache) and map.
|
||||
Benchmark tests were made using an i7-6700K with 32GB of RAM on Windows 10.
|
||||
|
||||
### Writes and reads
|
||||
|
||||
```bash
|
||||
cd caches_bench; go test -bench=. -benchtime=10s ./... -timeout 30m
|
||||
|
||||
BenchmarkMapSet-8 2000000 716 ns/op 336 B/op 3 allocs/op
|
||||
BenchmarkConcurrentMapSet-8 1000000 1292 ns/op 347 B/op 8 allocs/op
|
||||
BenchmarkFreeCacheSet-8 3000000 501 ns/op 371 B/op 3 allocs/op
|
||||
BenchmarkBigCacheSet-8 3000000 482 ns/op 303 B/op 2 allocs/op
|
||||
BenchmarkMapGet-8 5000000 309 ns/op 24 B/op 1 allocs/op
|
||||
BenchmarkConcurrentMapGet-8 2000000 659 ns/op 24 B/op 2 allocs/op
|
||||
BenchmarkFreeCacheGet-8 3000000 541 ns/op 152 B/op 3 allocs/op
|
||||
BenchmarkBigCacheGet-8 3000000 420 ns/op 152 B/op 3 allocs/op
|
||||
BenchmarkBigCacheSetParallel-8 10000000 184 ns/op 313 B/op 3 allocs/op
|
||||
BenchmarkFreeCacheSetParallel-8 10000000 195 ns/op 357 B/op 4 allocs/op
|
||||
BenchmarkConcurrentMapSetParallel-8 5000000 242 ns/op 200 B/op 6 allocs/op
|
||||
BenchmarkBigCacheGetParallel-8 20000000 100 ns/op 152 B/op 4 allocs/op
|
||||
BenchmarkFreeCacheGetParallel-8 10000000 133 ns/op 152 B/op 4 allocs/op
|
||||
BenchmarkConcurrentMapGetParallel-8 10000000 202 ns/op 24 B/op 2 allocs/op
|
||||
```
|
||||
|
||||
Writes and reads in bigcache are faster than in freecache.
|
||||
Writes to map are the slowest.
|
||||
|
||||
### GC pause time
|
||||
|
||||
```bash
|
||||
cd caches_bench; go run caches_gc_overhead_comparison.go
|
||||
|
||||
Number of entries: 20000000
|
||||
GC pause for bigcache: 5.8658ms
|
||||
GC pause for freecache: 32.4341ms
|
||||
GC pause for map: 52.9661ms
|
||||
```
|
||||
|
||||
Test shows how long are the GC pauses for caches filled with 20mln of entries.
|
||||
Bigcache and freecache have very similar GC pause time.
|
||||
It is clear that both reduce GC overhead in contrast to map
|
||||
which GC pause time took more than 10 seconds.
|
||||
|
||||
## How it works
|
||||
|
||||
BigCache relies on optimization presented in 1.5 version of Go ([issue-9477](https://github.com/golang/go/issues/9477)).
|
||||
This optimization states that if map without pointers in keys and values is used then GC will omit its content.
|
||||
Therefore BigCache uses `map[uint64]uint32` where keys are hashed and values are offsets of entries.
|
||||
|
||||
Entries are kept in bytes array, to omit GC again.
|
||||
Bytes array size can grow to gigabytes without impact on performance
|
||||
because GC will only see single pointer to it.
|
||||
|
||||
## Bigcache vs Freecache
|
||||
|
||||
Both caches provide the same core features but they reduce GC overhead in different ways.
|
||||
Bigcache relies on `map[uint64]uint32`, freecache implements its own mapping built on
|
||||
slices to reduce number of pointers.
|
||||
|
||||
Results from benchmark tests are presented above.
|
||||
One of the advantage of bigcache over freecache is that you don’t need to know
|
||||
the size of the cache in advance, because when bigcache is full,
|
||||
it can allocate additional memory for new entries instead of
|
||||
overwriting existing ones as freecache does currently.
|
||||
However hard max size in bigcache also can be set, check [HardMaxCacheSize](https://godoc.org/github.com/allegro/bigcache#Config).
|
||||
|
||||
## HTTP Server
|
||||
|
||||
This package also includes an easily deployable HTTP implementation of BigCache, which can be found in the [server](/server) package.
|
||||
|
||||
## More
|
||||
|
||||
Bigcache genesis is described in allegro.tech blog post: [writing a very fast cache service in Go](http://allegro.tech/2016/03/writing-fast-cache-service-in-go.html)
|
||||
|
||||
## License
|
||||
|
||||
BigCache is released under the Apache 2.0 license (see [LICENSE](LICENSE))
|
||||
-155
@@ -1,155 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
minimumEntriesInShard = 10 // Minimum number of entries in single shard
|
||||
)
|
||||
|
||||
// BigCache is fast, concurrent, evicting cache created to keep big number of entries without impact on performance.
|
||||
// It keeps entries on heap but omits GC for them. To achieve that operations on bytes arrays take place,
|
||||
// therefore entries (de)serialization in front of the cache will be needed in most use cases.
|
||||
type BigCache struct {
|
||||
shards []*cacheShard
|
||||
lifeWindow uint64
|
||||
clock clock
|
||||
hash Hasher
|
||||
config Config
|
||||
shardMask uint64
|
||||
maxShardSize uint32
|
||||
}
|
||||
|
||||
// NewBigCache initialize new instance of BigCache
|
||||
func NewBigCache(config Config) (*BigCache, error) {
|
||||
return newBigCache(config, &systemClock{})
|
||||
}
|
||||
|
||||
func newBigCache(config Config, clock clock) (*BigCache, error) {
|
||||
|
||||
if !isPowerOfTwo(config.Shards) {
|
||||
return nil, fmt.Errorf("Shards number must be power of two")
|
||||
}
|
||||
|
||||
if config.Hasher == nil {
|
||||
config.Hasher = newDefaultHasher()
|
||||
}
|
||||
|
||||
cache := &BigCache{
|
||||
shards: make([]*cacheShard, config.Shards),
|
||||
lifeWindow: uint64(config.LifeWindow.Seconds()),
|
||||
clock: clock,
|
||||
hash: config.Hasher,
|
||||
config: config,
|
||||
shardMask: uint64(config.Shards - 1),
|
||||
maxShardSize: uint32(config.maximumShardSize()),
|
||||
}
|
||||
|
||||
var onRemove func(wrappedEntry []byte)
|
||||
if config.OnRemove == nil {
|
||||
onRemove = cache.notProvidedOnRemove
|
||||
} else {
|
||||
onRemove = cache.providedOnRemove
|
||||
}
|
||||
|
||||
for i := 0; i < config.Shards; i++ {
|
||||
cache.shards[i] = initNewShard(config, onRemove, clock)
|
||||
}
|
||||
|
||||
if config.CleanWindow > 0 {
|
||||
go func() {
|
||||
for t := range time.Tick(config.CleanWindow) {
|
||||
cache.cleanUp(uint64(t.Unix()))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
// Get reads entry for the key.
|
||||
// It returns an EntryNotFoundError when
|
||||
// no entry exists for the given key.
|
||||
func (c *BigCache) Get(key string) ([]byte, error) {
|
||||
hashedKey := c.hash.Sum64(key)
|
||||
shard := c.getShard(hashedKey)
|
||||
return shard.get(key, hashedKey)
|
||||
}
|
||||
|
||||
// Set saves entry under the key
|
||||
func (c *BigCache) Set(key string, entry []byte) error {
|
||||
hashedKey := c.hash.Sum64(key)
|
||||
shard := c.getShard(hashedKey)
|
||||
return shard.set(key, hashedKey, entry)
|
||||
}
|
||||
|
||||
// Delete removes the key
|
||||
func (c *BigCache) Delete(key string) error {
|
||||
hashedKey := c.hash.Sum64(key)
|
||||
shard := c.getShard(hashedKey)
|
||||
return shard.del(key, hashedKey)
|
||||
}
|
||||
|
||||
// Reset empties all cache shards
|
||||
func (c *BigCache) Reset() error {
|
||||
for _, shard := range c.shards {
|
||||
shard.reset(c.config)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Len computes number of entries in cache
|
||||
func (c *BigCache) Len() int {
|
||||
var len int
|
||||
for _, shard := range c.shards {
|
||||
len += shard.len()
|
||||
}
|
||||
return len
|
||||
}
|
||||
|
||||
// Stats returns cache's statistics
|
||||
func (c *BigCache) Stats() Stats {
|
||||
var s Stats
|
||||
for _, shard := range c.shards {
|
||||
tmp := shard.getStats()
|
||||
s.Hits += tmp.Hits
|
||||
s.Misses += tmp.Misses
|
||||
s.DelHits += tmp.DelHits
|
||||
s.DelMisses += tmp.DelMisses
|
||||
s.Collisions += tmp.Collisions
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Iterator returns iterator function to iterate over EntryInfo's from whole cache.
|
||||
func (c *BigCache) Iterator() *EntryInfoIterator {
|
||||
return newIterator(c)
|
||||
}
|
||||
|
||||
func (c *BigCache) onEvict(oldestEntry []byte, currentTimestamp uint64, evict func() error) bool {
|
||||
oldestTimestamp := readTimestampFromEntry(oldestEntry)
|
||||
if currentTimestamp-oldestTimestamp > c.lifeWindow {
|
||||
evict()
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *BigCache) cleanUp(currentTimestamp uint64) {
|
||||
for _, shard := range c.shards {
|
||||
shard.cleanUp(currentTimestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *BigCache) getShard(hashedKey uint64) (shard *cacheShard) {
|
||||
return c.shards[hashedKey&c.shardMask]
|
||||
}
|
||||
|
||||
func (c *BigCache) providedOnRemove(wrappedEntry []byte) {
|
||||
c.config.OnRemove(readKeyFromEntry(wrappedEntry), readEntry(wrappedEntry))
|
||||
}
|
||||
|
||||
func (c *BigCache) notProvidedOnRemove(wrappedEntry []byte) {
|
||||
}
|
||||
-141
@@ -1,141 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var message = blob('a', 256)
|
||||
|
||||
func BenchmarkWriteToCacheWith1Shard(b *testing.B) {
|
||||
writeToCache(b, 1, 100*time.Second, b.N)
|
||||
}
|
||||
|
||||
func BenchmarkWriteToLimitedCacheWithSmallInitSizeAnd1Shard(b *testing.B) {
|
||||
m := blob('a', 1024)
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 100 * time.Second,
|
||||
MaxEntriesInWindow: 100,
|
||||
MaxEntrySize: 256,
|
||||
HardMaxCacheSize: 1,
|
||||
})
|
||||
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(fmt.Sprintf("key-%d", i), m)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWriteToUnlimitedCacheWithSmallInitSizeAnd1Shard(b *testing.B) {
|
||||
m := blob('a', 1024)
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 100 * time.Second,
|
||||
MaxEntriesInWindow: 100,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(fmt.Sprintf("key-%d", i), m)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWriteToCache(b *testing.B) {
|
||||
for _, shards := range []int{1, 512, 1024, 8192} {
|
||||
b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
|
||||
writeToCache(b, shards, 100*time.Second, b.N)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkReadFromCache(b *testing.B) {
|
||||
for _, shards := range []int{1, 512, 1024, 8192} {
|
||||
b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
|
||||
readFromCache(b, 1024)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIterateOverCache(b *testing.B) {
|
||||
|
||||
m := blob('a', 1)
|
||||
|
||||
for _, shards := range []int{512, 1024, 8192} {
|
||||
b.Run(fmt.Sprintf("%d-shards", shards), func(b *testing.B) {
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: shards,
|
||||
LifeWindow: 1000 * time.Second,
|
||||
MaxEntriesInWindow: max(b.N, 100),
|
||||
MaxEntrySize: 500,
|
||||
})
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(fmt.Sprintf("key-%d", i), m)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
it := cache.Iterator()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
b.ReportAllocs()
|
||||
|
||||
for pb.Next() {
|
||||
if it.SetNext() {
|
||||
it.Value()
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWriteToCacheWith1024ShardsAndSmallShardInitSize(b *testing.B) {
|
||||
writeToCache(b, 1024, 100*time.Second, 100)
|
||||
}
|
||||
|
||||
func writeToCache(b *testing.B, shards int, lifeWindow time.Duration, requestsInLifeWindow int) {
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: shards,
|
||||
LifeWindow: lifeWindow,
|
||||
MaxEntriesInWindow: max(requestsInLifeWindow, 100),
|
||||
MaxEntrySize: 500,
|
||||
})
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := rand.Int()
|
||||
counter := 0
|
||||
|
||||
b.ReportAllocs()
|
||||
for pb.Next() {
|
||||
cache.Set(fmt.Sprintf("key-%d-%d", id, counter), message)
|
||||
counter = counter + 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func readFromCache(b *testing.B, shards int) {
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: shards,
|
||||
LifeWindow: 1000 * time.Second,
|
||||
MaxEntriesInWindow: max(b.N, 100),
|
||||
MaxEntrySize: 500,
|
||||
})
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(strconv.Itoa(i), message)
|
||||
}
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
b.ReportAllocs()
|
||||
|
||||
for pb.Next() {
|
||||
cache.Get(strconv.Itoa(rand.Intn(b.N)))
|
||||
}
|
||||
})
|
||||
}
|
||||
-579
@@ -1,579 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var sink []byte
|
||||
|
||||
func TestParallel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(DefaultConfig(5 * time.Second))
|
||||
value := []byte("value")
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(3)
|
||||
keys := 1337
|
||||
|
||||
// when
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), value)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < keys; i++ {
|
||||
sink, _ = cache.Get(fmt.Sprintf("key%d", i))
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Delete(fmt.Sprintf("key%d", i))
|
||||
}
|
||||
}()
|
||||
|
||||
// then
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestWriteAndGetOnCache(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(DefaultConfig(5 * time.Second))
|
||||
value := []byte("value")
|
||||
|
||||
// when
|
||||
cache.Set("key", value)
|
||||
cachedValue, err := cache.Get("key")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, value, cachedValue)
|
||||
}
|
||||
|
||||
func TestConstructCacheWithDefaultHasher(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 16,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 10,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
|
||||
assert.IsType(t, fnv64a{}, cache.hash)
|
||||
}
|
||||
|
||||
func TestWillReturnErrorOnInvalidNumberOfPartitions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, error := NewBigCache(Config{
|
||||
Shards: 18,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 10,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
|
||||
assert.Nil(t, cache)
|
||||
assert.Error(t, error, "Shards number must be power of two")
|
||||
}
|
||||
|
||||
func TestEntryNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 16,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 10,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
|
||||
// when
|
||||
_, err := cache.Get("nonExistingKey")
|
||||
|
||||
// then
|
||||
assert.EqualError(t, err, "Entry \"nonExistingKey\" not found")
|
||||
}
|
||||
|
||||
func TestTimingEviction(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
clock := mockedClock{value: 0}
|
||||
cache, _ := newBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
}, &clock)
|
||||
|
||||
// when
|
||||
cache.Set("key", []byte("value"))
|
||||
clock.set(5)
|
||||
cache.Set("key2", []byte("value2"))
|
||||
_, err := cache.Get("key")
|
||||
|
||||
// then
|
||||
assert.EqualError(t, err, "Entry \"key\" not found")
|
||||
}
|
||||
|
||||
func TestTimingEvictionShouldEvictOnlyFromUpdatedShard(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
clock := mockedClock{value: 0}
|
||||
cache, _ := newBigCache(Config{
|
||||
Shards: 4,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
}, &clock)
|
||||
|
||||
// when
|
||||
cache.Set("key", []byte("value"))
|
||||
clock.set(5)
|
||||
cache.Set("key2", []byte("value 2"))
|
||||
value, err := cache.Get("key")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err, "Entry \"key\" not found")
|
||||
assert.Equal(t, []byte("value"), value)
|
||||
}
|
||||
|
||||
func TestCleanShouldEvictAll(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 4,
|
||||
LifeWindow: time.Second,
|
||||
CleanWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
|
||||
// when
|
||||
cache.Set("key", []byte("value"))
|
||||
<-time.After(3 * time.Second)
|
||||
value, err := cache.Get("key")
|
||||
|
||||
// then
|
||||
assert.EqualError(t, err, "Entry \"key\" not found")
|
||||
assert.Equal(t, value, []byte(nil))
|
||||
}
|
||||
|
||||
func TestOnRemoveCallback(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
clock := mockedClock{value: 0}
|
||||
onRemoveInvoked := false
|
||||
onRemove := func(key string, entry []byte) {
|
||||
onRemoveInvoked = true
|
||||
assert.Equal(t, "key", key)
|
||||
assert.Equal(t, []byte("value"), entry)
|
||||
}
|
||||
cache, _ := newBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
OnRemove: onRemove,
|
||||
}, &clock)
|
||||
|
||||
// when
|
||||
cache.Set("key", []byte("value"))
|
||||
clock.set(5)
|
||||
cache.Set("key2", []byte("value2"))
|
||||
|
||||
// then
|
||||
assert.True(t, onRemoveInvoked)
|
||||
}
|
||||
|
||||
func TestCacheLen(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 8,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
keys := 1337
|
||||
|
||||
// when
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
|
||||
}
|
||||
|
||||
// then
|
||||
assert.Equal(t, keys, cache.Len())
|
||||
}
|
||||
|
||||
func TestCacheStats(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 8,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
|
||||
// when
|
||||
for i := 0; i < 100; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
value, err := cache.Get(fmt.Sprintf("key%d", i))
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, string(value), "value")
|
||||
}
|
||||
for i := 100; i < 110; i++ {
|
||||
_, err := cache.Get(fmt.Sprintf("key%d", i))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
for i := 10; i < 20; i++ {
|
||||
err := cache.Delete(fmt.Sprintf("key%d", i))
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
for i := 110; i < 120; i++ {
|
||||
err := cache.Delete(fmt.Sprintf("key%d", i))
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// then
|
||||
stats := cache.Stats()
|
||||
assert.Equal(t, stats.Hits, int64(10))
|
||||
assert.Equal(t, stats.Misses, int64(10))
|
||||
assert.Equal(t, stats.DelHits, int64(10))
|
||||
assert.Equal(t, stats.DelMisses, int64(10))
|
||||
}
|
||||
|
||||
func TestCacheDel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(DefaultConfig(time.Second))
|
||||
|
||||
// when
|
||||
err := cache.Delete("nonExistingKey")
|
||||
|
||||
// then
|
||||
assert.Equal(t, err.Error(), "Entry \"nonExistingKey\" not found")
|
||||
|
||||
// and when
|
||||
cache.Set("existingKey", nil)
|
||||
err = cache.Delete("existingKey")
|
||||
cachedValue, _ := cache.Get("existingKey")
|
||||
|
||||
// then
|
||||
assert.Nil(t, err)
|
||||
assert.Len(t, cachedValue, 0)
|
||||
}
|
||||
|
||||
func TestCacheReset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 8,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
keys := 1337
|
||||
|
||||
// when
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
|
||||
}
|
||||
|
||||
// then
|
||||
assert.Equal(t, keys, cache.Len())
|
||||
|
||||
// and when
|
||||
cache.Reset()
|
||||
|
||||
// then
|
||||
assert.Equal(t, 0, cache.Len())
|
||||
|
||||
// and when
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
|
||||
}
|
||||
|
||||
// then
|
||||
assert.Equal(t, keys, cache.Len())
|
||||
}
|
||||
|
||||
func TestIterateOnResetCache(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 8,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
keys := 1337
|
||||
|
||||
// when
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
|
||||
}
|
||||
cache.Reset()
|
||||
|
||||
// then
|
||||
iterator := cache.Iterator()
|
||||
|
||||
assert.Equal(t, false, iterator.SetNext())
|
||||
}
|
||||
|
||||
func TestGetOnResetCache(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 8,
|
||||
LifeWindow: time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
})
|
||||
keys := 1337
|
||||
|
||||
// when
|
||||
for i := 0; i < keys; i++ {
|
||||
cache.Set(fmt.Sprintf("key%d", i), []byte("value"))
|
||||
}
|
||||
|
||||
cache.Reset()
|
||||
|
||||
// then
|
||||
value, err := cache.Get("key1")
|
||||
|
||||
assert.Equal(t, err.Error(), "Entry \"key1\" not found")
|
||||
assert.Equal(t, value, []byte(nil))
|
||||
}
|
||||
|
||||
func TestEntryUpdate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
clock := mockedClock{value: 0}
|
||||
cache, _ := newBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 6 * time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 256,
|
||||
}, &clock)
|
||||
|
||||
// when
|
||||
cache.Set("key", []byte("value"))
|
||||
clock.set(5)
|
||||
cache.Set("key", []byte("value2"))
|
||||
clock.set(7)
|
||||
cache.Set("key2", []byte("value3"))
|
||||
cachedValue, _ := cache.Get("key")
|
||||
|
||||
// then
|
||||
assert.Equal(t, []byte("value2"), cachedValue)
|
||||
}
|
||||
|
||||
func TestOldestEntryDeletionWhenMaxCacheSizeIsReached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 1,
|
||||
HardMaxCacheSize: 1,
|
||||
})
|
||||
|
||||
// when
|
||||
cache.Set("key1", blob('a', 1024*400))
|
||||
cache.Set("key2", blob('b', 1024*400))
|
||||
cache.Set("key3", blob('c', 1024*800))
|
||||
|
||||
_, key1Err := cache.Get("key1")
|
||||
_, key2Err := cache.Get("key2")
|
||||
entry3, _ := cache.Get("key3")
|
||||
|
||||
// then
|
||||
assert.EqualError(t, key1Err, "Entry \"key1\" not found")
|
||||
assert.EqualError(t, key2Err, "Entry \"key2\" not found")
|
||||
assert.Equal(t, blob('c', 1024*800), entry3)
|
||||
}
|
||||
|
||||
func TestRetrievingEntryShouldCopy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 1,
|
||||
HardMaxCacheSize: 1,
|
||||
})
|
||||
cache.Set("key1", blob('a', 1024*400))
|
||||
value, key1Err := cache.Get("key1")
|
||||
|
||||
// when
|
||||
// override queue
|
||||
cache.Set("key2", blob('b', 1024*400))
|
||||
cache.Set("key3", blob('c', 1024*400))
|
||||
cache.Set("key4", blob('d', 1024*400))
|
||||
cache.Set("key5", blob('d', 1024*400))
|
||||
|
||||
// then
|
||||
assert.Nil(t, key1Err)
|
||||
assert.Equal(t, blob('a', 1024*400), value)
|
||||
}
|
||||
|
||||
func TestEntryBiggerThanMaxShardSizeError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 1,
|
||||
HardMaxCacheSize: 1,
|
||||
})
|
||||
|
||||
// when
|
||||
err := cache.Set("key1", blob('a', 1024*1025))
|
||||
|
||||
// then
|
||||
assert.EqualError(t, err, "entry is bigger than max shard size")
|
||||
}
|
||||
|
||||
func TestHashCollision(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ml := &mockedLogger{}
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 16,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 10,
|
||||
MaxEntrySize: 256,
|
||||
Verbose: true,
|
||||
Hasher: hashStub(5),
|
||||
Logger: ml,
|
||||
})
|
||||
|
||||
// when
|
||||
cache.Set("liquid", []byte("value"))
|
||||
cachedValue, err := cache.Get("liquid")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("value"), cachedValue)
|
||||
|
||||
// when
|
||||
cache.Set("costarring", []byte("value 2"))
|
||||
cachedValue, err = cache.Get("costarring")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte("value 2"), cachedValue)
|
||||
|
||||
// when
|
||||
cachedValue, err = cache.Get("liquid")
|
||||
|
||||
// then
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, cachedValue)
|
||||
|
||||
assert.NotEqual(t, "", ml.lastFormat)
|
||||
assert.Equal(t, cache.Stats().Collisions, int64(1))
|
||||
}
|
||||
|
||||
func TestNilValueCaching(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// given
|
||||
cache, _ := NewBigCache(Config{
|
||||
Shards: 1,
|
||||
LifeWindow: 5 * time.Second,
|
||||
MaxEntriesInWindow: 1,
|
||||
MaxEntrySize: 1,
|
||||
HardMaxCacheSize: 1,
|
||||
})
|
||||
|
||||
// when
|
||||
cache.Set("Kierkegaard", []byte{})
|
||||
cachedValue, err := cache.Get("Kierkegaard")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte{}, cachedValue)
|
||||
|
||||
// when
|
||||
cache.Set("Sartre", nil)
|
||||
cachedValue, err = cache.Get("Sartre")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte{}, cachedValue)
|
||||
|
||||
// when
|
||||
cache.Set("Nietzsche", []byte(nil))
|
||||
cachedValue, err = cache.Get("Nietzsche")
|
||||
|
||||
// then
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte{}, cachedValue)
|
||||
}
|
||||
|
||||
type mockedLogger struct {
|
||||
lastFormat string
|
||||
lastArgs []interface{}
|
||||
}
|
||||
|
||||
func (ml *mockedLogger) Printf(format string, v ...interface{}) {
|
||||
ml.lastFormat = format
|
||||
ml.lastArgs = v
|
||||
}
|
||||
|
||||
type mockedClock struct {
|
||||
value int64
|
||||
}
|
||||
|
||||
func (mc *mockedClock) epoch() int64 {
|
||||
return mc.value
|
||||
}
|
||||
|
||||
func (mc *mockedClock) set(value int64) {
|
||||
mc.value = value
|
||||
}
|
||||
|
||||
func blob(char byte, len int) []byte {
|
||||
b := make([]byte, len)
|
||||
for index := range b {
|
||||
b[index] = char
|
||||
}
|
||||
return b
|
||||
}
|
||||
-219
@@ -1,219 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/allegro/bigcache"
|
||||
"github.com/coocood/freecache"
|
||||
)
|
||||
|
||||
const maxEntrySize = 256
|
||||
|
||||
func BenchmarkMapSet(b *testing.B) {
|
||||
m := make(map[string][]byte)
|
||||
for i := 0; i < b.N; i++ {
|
||||
m[key(i)] = value()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConcurrentMapSet(b *testing.B) {
|
||||
var m sync.Map
|
||||
for i := 0; i < b.N; i++ {
|
||||
m.Store(key(i), value())
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFreeCacheSet(b *testing.B) {
|
||||
cache := freecache.NewCache(b.N * maxEntrySize)
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set([]byte(key(i)), value(), 0)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBigCacheSet(b *testing.B) {
|
||||
cache := initBigCache(b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(key(i), value())
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMapGet(b *testing.B) {
|
||||
b.StopTimer()
|
||||
m := make(map[string][]byte)
|
||||
for i := 0; i < b.N; i++ {
|
||||
m[key(i)] = value()
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
hitCount := 0
|
||||
for i := 0; i < b.N; i++ {
|
||||
if m[key(i)] != nil {
|
||||
hitCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkConcurrentMapGet(b *testing.B) {
|
||||
b.StopTimer()
|
||||
var m sync.Map
|
||||
for i := 0; i < b.N; i++ {
|
||||
m.Store(key(i), value())
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
hitCounter := 0
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, ok := m.Load(key(i))
|
||||
if ok {
|
||||
hitCounter++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFreeCacheGet(b *testing.B) {
|
||||
b.StopTimer()
|
||||
cache := freecache.NewCache(b.N * maxEntrySize)
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set([]byte(key(i)), value(), 0)
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Get([]byte(key(i)))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBigCacheGet(b *testing.B) {
|
||||
b.StopTimer()
|
||||
cache := initBigCache(b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(key(i), value())
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Get(key(i))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBigCacheSetParallel(b *testing.B) {
|
||||
cache := initBigCache(b.N)
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := rand.Intn(1000)
|
||||
counter := 0
|
||||
for pb.Next() {
|
||||
cache.Set(parallelKey(id, counter), value())
|
||||
counter = counter + 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkFreeCacheSetParallel(b *testing.B) {
|
||||
cache := freecache.NewCache(b.N * maxEntrySize)
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := rand.Intn(1000)
|
||||
counter := 0
|
||||
for pb.Next() {
|
||||
cache.Set([]byte(parallelKey(id, counter)), value(), 0)
|
||||
counter = counter + 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkConcurrentMapSetParallel(b *testing.B) {
|
||||
var m sync.Map
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := rand.Intn(1000)
|
||||
for pb.Next() {
|
||||
m.Store(key(id), value())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkBigCacheGetParallel(b *testing.B) {
|
||||
b.StopTimer()
|
||||
cache := initBigCache(b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set(key(i), value())
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
counter := 0
|
||||
for pb.Next() {
|
||||
cache.Get(key(counter))
|
||||
counter = counter + 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkFreeCacheGetParallel(b *testing.B) {
|
||||
b.StopTimer()
|
||||
cache := freecache.NewCache(b.N * maxEntrySize)
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Set([]byte(key(i)), value(), 0)
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
counter := 0
|
||||
for pb.Next() {
|
||||
cache.Get([]byte(key(counter)))
|
||||
counter = counter + 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkConcurrentMapGetParallel(b *testing.B) {
|
||||
b.StopTimer()
|
||||
var m sync.Map
|
||||
for i := 0; i < b.N; i++ {
|
||||
m.Store(key(i), value())
|
||||
}
|
||||
|
||||
b.StartTimer()
|
||||
hitCount := 0
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := rand.Intn(1000)
|
||||
for pb.Next() {
|
||||
_, ok := m.Load(key(id))
|
||||
if ok {
|
||||
hitCount++
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func key(i int) string {
|
||||
return fmt.Sprintf("key-%010d", i)
|
||||
}
|
||||
|
||||
func value() []byte {
|
||||
return make([]byte, 100)
|
||||
}
|
||||
|
||||
func parallelKey(threadID int, counter int) string {
|
||||
return fmt.Sprintf("key-%04d-%06d", threadID, counter)
|
||||
}
|
||||
|
||||
func initBigCache(entriesInWindow int) *bigcache.BigCache {
|
||||
cache, _ := bigcache.NewBigCache(bigcache.Config{
|
||||
Shards: 256,
|
||||
LifeWindow: 10 * time.Minute,
|
||||
MaxEntriesInWindow: entriesInWindow,
|
||||
MaxEntrySize: maxEntrySize,
|
||||
Verbose: true,
|
||||
})
|
||||
|
||||
return cache
|
||||
}
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/allegro/bigcache"
|
||||
"github.com/coocood/freecache"
|
||||
)
|
||||
|
||||
func gcPause() time.Duration {
|
||||
runtime.GC()
|
||||
var stats debug.GCStats
|
||||
debug.ReadGCStats(&stats)
|
||||
return stats.PauseTotal
|
||||
}
|
||||
|
||||
const (
|
||||
entries = 20000000
|
||||
valueSize = 100
|
||||
)
|
||||
|
||||
func main() {
|
||||
debug.SetGCPercent(10)
|
||||
fmt.Println("Number of entries: ", entries)
|
||||
|
||||
config := bigcache.Config{
|
||||
Shards: 256,
|
||||
LifeWindow: 100 * time.Minute,
|
||||
MaxEntriesInWindow: entries,
|
||||
MaxEntrySize: 200,
|
||||
Verbose: true,
|
||||
}
|
||||
|
||||
bigcache, _ := bigcache.NewBigCache(config)
|
||||
for i := 0; i < entries; i++ {
|
||||
key, val := generateKeyValue(i, valueSize)
|
||||
bigcache.Set(key, val)
|
||||
}
|
||||
|
||||
firstKey, _ := generateKeyValue(1, valueSize)
|
||||
checkFirstElement(bigcache.Get(firstKey))
|
||||
|
||||
fmt.Println("GC pause for bigcache: ", gcPause())
|
||||
bigcache = nil
|
||||
gcPause()
|
||||
|
||||
//------------------------------------------
|
||||
|
||||
freeCache := freecache.NewCache(entries * 200) //allocate entries * 200 bytes
|
||||
for i := 0; i < entries; i++ {
|
||||
key, val := generateKeyValue(i, valueSize)
|
||||
if err := freeCache.Set([]byte(key), val, 0); err != nil {
|
||||
fmt.Println("Error in set: ", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
firstKey, _ = generateKeyValue(1, valueSize)
|
||||
checkFirstElement(freeCache.Get([]byte(firstKey)))
|
||||
|
||||
if freeCache.OverwriteCount() != 0 {
|
||||
fmt.Println("Overwritten: ", freeCache.OverwriteCount())
|
||||
}
|
||||
fmt.Println("GC pause for freecache: ", gcPause())
|
||||
freeCache = nil
|
||||
gcPause()
|
||||
|
||||
//------------------------------------------
|
||||
|
||||
mapCache := make(map[string][]byte)
|
||||
for i := 0; i < entries; i++ {
|
||||
key, val := generateKeyValue(i, valueSize)
|
||||
mapCache[key] = val
|
||||
}
|
||||
fmt.Println("GC pause for map: ", gcPause())
|
||||
|
||||
}
|
||||
|
||||
func checkFirstElement(val []byte, err error) {
|
||||
_, expectedVal := generateKeyValue(1, valueSize)
|
||||
if err != nil {
|
||||
fmt.Println("Error in get: ", err.Error())
|
||||
} else if string(val) != string(expectedVal) {
|
||||
fmt.Println("Wrong first element: ", string(val))
|
||||
}
|
||||
}
|
||||
|
||||
func generateKeyValue(index int, valSize int) (string, []byte) {
|
||||
key := fmt.Sprintf("key-%010d", index)
|
||||
fixedNumber := []byte(fmt.Sprintf("%010d", index))
|
||||
val := append(make([]byte, valSize-10), fixedNumber...)
|
||||
|
||||
return key, val
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import "time"
|
||||
|
||||
type clock interface {
|
||||
epoch() int64
|
||||
}
|
||||
|
||||
type systemClock struct {
|
||||
}
|
||||
|
||||
func (c systemClock) epoch() int64 {
|
||||
return time.Now().Unix()
|
||||
}
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import "time"
|
||||
|
||||
// Config for BigCache
|
||||
type Config struct {
|
||||
// Number of cache shards, value must be a power of two
|
||||
Shards int
|
||||
// Time after which entry can be evicted
|
||||
LifeWindow time.Duration
|
||||
// Interval between removing expired entries (clean up).
|
||||
// If set to <= 0 then no action is performed. Setting to < 1 second is counterproductive — bigcache has a one second resolution.
|
||||
CleanWindow time.Duration
|
||||
// Max number of entries in life window. Used only to calculate initial size for cache shards.
|
||||
// When proper value is set then additional memory allocation does not occur.
|
||||
MaxEntriesInWindow int
|
||||
// Max size of entry in bytes. Used only to calculate initial size for cache shards.
|
||||
MaxEntrySize int
|
||||
// Verbose mode prints information about new memory allocation
|
||||
Verbose bool
|
||||
// Hasher used to map between string keys and unsigned 64bit integers, by default fnv64 hashing is used.
|
||||
Hasher Hasher
|
||||
// HardMaxCacheSize is a limit for cache size in MB. Cache will not allocate more memory than this limit.
|
||||
// It can protect application from consuming all available memory on machine, therefore from running OOM Killer.
|
||||
// Default value is 0 which means unlimited size. When the limit is higher than 0 and reached then
|
||||
// the oldest entries are overridden for the new ones.
|
||||
HardMaxCacheSize int
|
||||
// OnRemove is a callback fired when the oldest entry is removed because of its expiration time or no space left
|
||||
// for the new entry. Default value is nil which means no callback and it prevents from unwrapping the oldest entry.
|
||||
OnRemove func(key string, entry []byte)
|
||||
|
||||
// Logger is a logging interface and used in combination with `Verbose`
|
||||
// Defaults to `DefaultLogger()`
|
||||
Logger Logger
|
||||
}
|
||||
|
||||
// DefaultConfig initializes config with default values.
|
||||
// When load for BigCache can be predicted in advance then it is better to use custom config.
|
||||
func DefaultConfig(eviction time.Duration) Config {
|
||||
return Config{
|
||||
Shards: 1024,
|
||||
LifeWindow: eviction,
|
||||
CleanWindow: 0,
|
||||
MaxEntriesInWindow: 1000 * 10 * 60,
|
||||
MaxEntrySize: 500,
|
||||
Verbose: true,
|
||||
Hasher: newDefaultHasher(),
|
||||
HardMaxCacheSize: 0,
|
||||
Logger: DefaultLogger(),
|
||||
}
|
||||
}
|
||||
|
||||
// initialShardSize computes initial shard size
|
||||
func (c Config) initialShardSize() int {
|
||||
return max(c.MaxEntriesInWindow/c.Shards, minimumEntriesInShard)
|
||||
}
|
||||
|
||||
// maximumShardSize computes maximum shard size
|
||||
func (c Config) maximumShardSize() int {
|
||||
maxShardSize := 0
|
||||
|
||||
if c.HardMaxCacheSize > 0 {
|
||||
maxShardSize = convertMBToBytes(c.HardMaxCacheSize) / c.Shards
|
||||
}
|
||||
|
||||
return maxShardSize
|
||||
}
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
timestampSizeInBytes = 8 // Number of bytes used for timestamp
|
||||
hashSizeInBytes = 8 // Number of bytes used for hash
|
||||
keySizeInBytes = 2 // Number of bytes used for size of entry key
|
||||
headersSizeInBytes = timestampSizeInBytes + hashSizeInBytes + keySizeInBytes // Number of bytes used for all headers
|
||||
)
|
||||
|
||||
func wrapEntry(timestamp uint64, hash uint64, key string, entry []byte, buffer *[]byte) []byte {
|
||||
keyLength := len(key)
|
||||
blobLength := len(entry) + headersSizeInBytes + keyLength
|
||||
|
||||
if blobLength > len(*buffer) {
|
||||
*buffer = make([]byte, blobLength)
|
||||
}
|
||||
blob := *buffer
|
||||
|
||||
binary.LittleEndian.PutUint64(blob, timestamp)
|
||||
binary.LittleEndian.PutUint64(blob[timestampSizeInBytes:], hash)
|
||||
binary.LittleEndian.PutUint16(blob[timestampSizeInBytes+hashSizeInBytes:], uint16(keyLength))
|
||||
copy(blob[headersSizeInBytes:], key)
|
||||
copy(blob[headersSizeInBytes+keyLength:], entry)
|
||||
|
||||
return blob[:blobLength]
|
||||
}
|
||||
|
||||
func readEntry(data []byte) []byte {
|
||||
length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:])
|
||||
|
||||
// copy on read
|
||||
dst := make([]byte, len(data)-int(headersSizeInBytes+length))
|
||||
copy(dst, data[headersSizeInBytes+length:])
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
func readTimestampFromEntry(data []byte) uint64 {
|
||||
return binary.LittleEndian.Uint64(data)
|
||||
}
|
||||
|
||||
func readKeyFromEntry(data []byte) string {
|
||||
length := binary.LittleEndian.Uint16(data[timestampSizeInBytes+hashSizeInBytes:])
|
||||
|
||||
// copy on read
|
||||
dst := make([]byte, length)
|
||||
copy(dst, data[headersSizeInBytes:headersSizeInBytes+length])
|
||||
|
||||
return bytesToString(dst)
|
||||
}
|
||||
|
||||
func bytesToString(b []byte) string {
|
||||
bytesHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
strHeader := reflect.StringHeader{Data: bytesHeader.Data, Len: bytesHeader.Len}
|
||||
return *(*string)(unsafe.Pointer(&strHeader))
|
||||
}
|
||||
|
||||
func readHashFromEntry(data []byte) uint64 {
|
||||
return binary.LittleEndian.Uint64(data[timestampSizeInBytes:])
|
||||
}
|
||||
|
||||
func resetKeyFromEntry(data []byte) {
|
||||
binary.LittleEndian.PutUint64(data[timestampSizeInBytes:], 0)
|
||||
}
|
||||
-46
@@ -1,46 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestEncodeDecode(t *testing.T) {
|
||||
// given
|
||||
now := uint64(time.Now().Unix())
|
||||
hash := uint64(42)
|
||||
key := "key"
|
||||
data := []byte("data")
|
||||
buffer := make([]byte, 100)
|
||||
|
||||
// when
|
||||
wrapped := wrapEntry(now, hash, key, data, &buffer)
|
||||
|
||||
// then
|
||||
assert.Equal(t, key, readKeyFromEntry(wrapped))
|
||||
assert.Equal(t, hash, readHashFromEntry(wrapped))
|
||||
assert.Equal(t, now, readTimestampFromEntry(wrapped))
|
||||
assert.Equal(t, data, readEntry(wrapped))
|
||||
assert.Equal(t, 100, len(buffer))
|
||||
}
|
||||
|
||||
func TestAllocateBiggerBuffer(t *testing.T) {
|
||||
//given
|
||||
now := uint64(time.Now().Unix())
|
||||
hash := uint64(42)
|
||||
key := "1"
|
||||
data := []byte("2")
|
||||
buffer := make([]byte, 1)
|
||||
|
||||
// when
|
||||
wrapped := wrapEntry(now, hash, key, data, &buffer)
|
||||
|
||||
// then
|
||||
assert.Equal(t, key, readKeyFromEntry(wrapped))
|
||||
assert.Equal(t, hash, readHashFromEntry(wrapped))
|
||||
assert.Equal(t, now, readTimestampFromEntry(wrapped))
|
||||
assert.Equal(t, data, readEntry(wrapped))
|
||||
assert.Equal(t, 2+headersSizeInBytes, len(buffer))
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import "fmt"
|
||||
|
||||
// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key
|
||||
type EntryNotFoundError struct {
|
||||
message string
|
||||
}
|
||||
|
||||
func notFound(key string) error {
|
||||
return &EntryNotFoundError{fmt.Sprintf("Entry %q not found", key)}
|
||||
}
|
||||
|
||||
// Error returned when entry does not exist.
|
||||
func (e EntryNotFoundError) Error() string {
|
||||
return e.message
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
// newDefaultHasher returns a new 64-bit FNV-1a Hasher which makes no memory allocations.
|
||||
// Its Sum64 method will lay the value out in big-endian byte order.
|
||||
// See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function
|
||||
func newDefaultHasher() Hasher {
|
||||
return fnv64a{}
|
||||
}
|
||||
|
||||
type fnv64a struct{}
|
||||
|
||||
const (
|
||||
// offset64 FNVa offset basis. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash
|
||||
offset64 = 14695981039346656037
|
||||
// prime64 FNVa prime value. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash
|
||||
prime64 = 1099511628211
|
||||
)
|
||||
|
||||
// Sum64 gets the string and returns its uint64 hash value.
|
||||
func (f fnv64a) Sum64(key string) uint64 {
|
||||
var hash uint64 = offset64
|
||||
for i := 0; i < len(key); i++ {
|
||||
hash ^= uint64(key[i])
|
||||
hash *= prime64
|
||||
}
|
||||
|
||||
return hash
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import "testing"
|
||||
|
||||
var text = "abcdefg"
|
||||
|
||||
func BenchmarkFnvHashSum64(b *testing.B) {
|
||||
h := newDefaultHasher()
|
||||
for i := 0; i < b.N; i++ {
|
||||
h.Sum64(text)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFnvHashStdLibSum64(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
stdLibFnvSum64(text)
|
||||
}
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
import (
|
||||
"hash/fnv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testCase struct {
|
||||
text string
|
||||
expectedHash uint64
|
||||
}
|
||||
|
||||
var testCases = []testCase{
|
||||
{"", stdLibFnvSum64("")},
|
||||
{"a", stdLibFnvSum64("a")},
|
||||
{"ab", stdLibFnvSum64("ab")},
|
||||
{"abc", stdLibFnvSum64("abc")},
|
||||
{"some longer and more complicated text", stdLibFnvSum64("some longer and more complicated text")},
|
||||
}
|
||||
|
||||
func TestFnvHashSum64(t *testing.T) {
|
||||
h := newDefaultHasher()
|
||||
for _, testCase := range testCases {
|
||||
hashed := h.Sum64(testCase.text)
|
||||
if hashed != testCase.expectedHash {
|
||||
t.Errorf("hash(%q) = %d want %d", testCase.text, hashed, testCase.expectedHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stdLibFnvSum64(key string) uint64 {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(key))
|
||||
return h.Sum64()
|
||||
}
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
// Hasher is responsible for generating unsigned, 64 bit hash of provided string. Hasher should minimize collisions
|
||||
// (generating same hash for different strings) and while performance is also important fast functions are preferable (i.e.
|
||||
// you can use FarmHash family).
|
||||
type Hasher interface {
|
||||
Sum64(string) uint64
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
package bigcache
|
||||
|
||||
type hashStub uint64
|
||||
|
||||
func (stub hashStub) Sum64(_ string) uint64 {
|
||||
return uint64(stub)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user