Vdb 269- fetch logs by hash (#122)

* Upgrade geth from 1.8.15 to 1.8.18

* Update vat_tune to use shared repository methods

* Query blockchain by block hash instead of block number range

* Remove hash validation from repositories

* Fix vow flog integration test

* Update README Travis build sticker

* Update constants formatting per go fmt

* Update EthPublicKeyParser.ParsePublicKey to use discv5.PubkeyID method

* Address PR comments
This commit is contained in:
Elizabeth 2018-11-21 09:30:00 -06:00 committed by GitHub
parent b19e77e419
commit 128d20c9bf
1328 changed files with 25048 additions and 210981 deletions

175
Gopkg.lock generated
View File

@ -3,30 +3,23 @@
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
name = "github.com/aristanetworks/goarista" name = "github.com/aristanetworks/goarista"
packages = ["monotime"] packages = ["monotime"]
pruneopts = ""
revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844" revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
name = "github.com/btcsuite/btcd" name = "github.com/btcsuite/btcd"
packages = ["btcec"] packages = ["btcec"]
pruneopts = ""
revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f" revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
[[projects]] [[projects]]
digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
name = "github.com/deckarep/golang-set" name = "github.com/deckarep/golang-set"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e" revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
version = "v1.7.1" version = "v1.7.1"
[[projects]] [[projects]]
digest = "1:8acdd83e35b407d026e546bf4d18cc778e9604433d03d468eccca62b9c51c718"
name = "github.com/ethereum/go-ethereum" name = "github.com/ethereum/go-ethereum"
packages = [ packages = [
".", ".",
@ -35,9 +28,11 @@
"accounts/abi/bind", "accounts/abi/bind",
"accounts/keystore", "accounts/keystore",
"common", "common",
"common/bitutil",
"common/hexutil", "common/hexutil",
"common/math", "common/math",
"common/mclock", "common/mclock",
"common/prque",
"core/rawdb", "core/rawdb",
"core/types", "core/types",
"crypto", "crypto",
@ -52,59 +47,49 @@
"p2p", "p2p",
"p2p/discover", "p2p/discover",
"p2p/discv5", "p2p/discv5",
"p2p/enode",
"p2p/enr",
"p2p/nat", "p2p/nat",
"p2p/netutil", "p2p/netutil",
"params", "params",
"rlp", "rlp",
"rpc", "rpc",
"trie", "trie"
] ]
pruneopts = "" revision = "58632d44021bf095b43a1bb2443e6e3690a94739"
revision = "89451f7c382ad2185987ee369f16416f89c28a7d" version = "v1.8.18"
version = "v1.8.15"
[[projects]] [[projects]]
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
name = "github.com/fsnotify/fsnotify" name = "github.com/fsnotify/fsnotify"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
version = "v1.4.7" version = "v1.4.7"
[[projects]] [[projects]]
digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
name = "github.com/go-stack/stack" name = "github.com/go-stack/stack"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a" revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
version = "v1.8.0" version = "v1.8.0"
[[projects]] [[projects]]
digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
name = "github.com/golang/protobuf" name = "github.com/golang/protobuf"
packages = ["proto"] packages = ["proto"]
pruneopts = ""
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
version = "v1.2.0" version = "v1.2.0"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
name = "github.com/golang/snappy" name = "github.com/golang/snappy"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
[[projects]] [[projects]]
digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
name = "github.com/google/uuid" name = "github.com/google/uuid"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494" revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
name = "github.com/hashicorp/hcl" name = "github.com/hashicorp/hcl"
packages = [ packages = [
".", ".",
@ -116,29 +101,25 @@
"hcl/token", "hcl/token",
"json/parser", "json/parser",
"json/scanner", "json/scanner",
"json/token", "json/token"
] ]
pruneopts = ""
revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
name = "github.com/hpcloud/tail" name = "github.com/hpcloud/tail"
packages = [ packages = [
".", ".",
"ratelimiter", "ratelimiter",
"util", "util",
"watch", "watch",
"winfile", "winfile"
] ]
pruneopts = ""
revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5" revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
name = "github.com/huin/goupnp" name = "github.com/huin/goupnp"
packages = [ packages = [
".", ".",
@ -147,75 +128,59 @@
"httpu", "httpu",
"scpd", "scpd",
"soap", "soap",
"ssdp", "ssdp"
] ]
pruneopts = ""
revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca" revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
[[projects]] [[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap" name = "github.com/inconshreveable/mousetrap"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0" version = "v1.0"
[[projects]] [[projects]]
digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
name = "github.com/jackpal/go-nat-pmp" name = "github.com/jackpal/go-nat-pmp"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62" revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
version = "v1.0.1" version = "v1.0.1"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
name = "github.com/jmoiron/sqlx" name = "github.com/jmoiron/sqlx"
packages = [ packages = [
".", ".",
"reflectx", "reflectx"
] ]
pruneopts = ""
revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849" revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
name = "github.com/lib/pq" name = "github.com/lib/pq"
packages = [ packages = [
".", ".",
"oid", "oid"
] ]
pruneopts = ""
revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79" revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
[[projects]] [[projects]]
digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
name = "github.com/magiconair/properties" name = "github.com/magiconair/properties"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "c2353362d570a7bfa228149c62842019201cfb71" revision = "c2353362d570a7bfa228149c62842019201cfb71"
version = "v1.8.0" version = "v1.8.0"
[[projects]] [[projects]]
digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
name = "github.com/mitchellh/go-homedir" name = "github.com/mitchellh/go-homedir"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4" revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
name = "github.com/mitchellh/mapstructure" name = "github.com/mitchellh/mapstructure"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8" revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
name = "github.com/onsi/ginkgo" name = "github.com/onsi/ginkgo"
packages = [ packages = [
".", ".",
@ -235,14 +200,12 @@
"reporters/stenographer", "reporters/stenographer",
"reporters/stenographer/support/go-colorable", "reporters/stenographer/support/go-colorable",
"reporters/stenographer/support/go-isatty", "reporters/stenographer/support/go-isatty",
"types", "types"
] ]
pruneopts = ""
revision = "3774a09d95489ccaa16032e0770d08ea77ba6184" revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
version = "v1.6.0" version = "v1.6.0"
[[projects]] [[projects]]
digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
name = "github.com/onsi/gomega" name = "github.com/onsi/gomega"
packages = [ packages = [
".", ".",
@ -257,106 +220,82 @@
"matchers/support/goraph/edge", "matchers/support/goraph/edge",
"matchers/support/goraph/node", "matchers/support/goraph/node",
"matchers/support/goraph/util", "matchers/support/goraph/util",
"types", "types"
] ]
pruneopts = ""
revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369" revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
version = "v1.4.2" version = "v1.4.2"
[[projects]] [[projects]]
digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
name = "github.com/pborman/uuid" name = "github.com/pborman/uuid"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1" revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
version = "v1.2" version = "v1.2"
[[projects]] [[projects]]
digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
name = "github.com/pelletier/go-toml" name = "github.com/pelletier/go-toml"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
version = "v1.2.0" version = "v1.2.0"
[[projects]] [[projects]]
digest = "1:210286d0cb60ffe28f1ca00b664029e8943009f95d06d8f8c336301b28e1aee5"
name = "github.com/philhofer/fwd" name = "github.com/philhofer/fwd"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "bb6d471dc95d4fe11e432687f8b70ff496cf3136" revision = "bb6d471dc95d4fe11e432687f8b70ff496cf3136"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
name = "github.com/rjeczalik/notify" name = "github.com/rjeczalik/notify"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8" revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
version = "v0.9.1" version = "v0.9.1"
[[projects]] [[projects]]
digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
name = "github.com/rs/cors" name = "github.com/rs/cors"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f" revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
version = "v1.5.0" version = "v1.5.0"
[[projects]] [[projects]]
digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
name = "github.com/spf13/afero" name = "github.com/spf13/afero"
packages = [ packages = [
".", ".",
"mem", "mem"
] ]
pruneopts = ""
revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd" revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
version = "v1.1.2" version = "v1.1.2"
[[projects]] [[projects]]
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
name = "github.com/spf13/cast" name = "github.com/spf13/cast"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "8965335b8c7107321228e3e3702cab9832751bac" revision = "8965335b8c7107321228e3e3702cab9832751bac"
version = "v1.2.0" version = "v1.2.0"
[[projects]] [[projects]]
digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
name = "github.com/spf13/cobra" name = "github.com/spf13/cobra"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
version = "v0.0.3" version = "v0.0.3"
[[projects]] [[projects]]
digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
name = "github.com/spf13/jwalterweatherman" name = "github.com/spf13/jwalterweatherman"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "4a4406e478ca629068e7768fc33f3f044173c0a6" revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
version = "v1.0.0" version = "v1.0.0"
[[projects]] [[projects]]
digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
name = "github.com/spf13/pflag" name = "github.com/spf13/pflag"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "9a97c102cda95a86cec2345a6f09f55a939babf5" revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
version = "v1.0.2" version = "v1.0.2"
[[projects]] [[projects]]
digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
name = "github.com/spf13/viper" name = "github.com/spf13/viper"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "8fb642006536c8d3760c99d4fa2389f5e2205631" revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
version = "v1.2.0" version = "v1.2.0"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
name = "github.com/syndtr/goleveldb" name = "github.com/syndtr/goleveldb"
packages = [ packages = [
"leveldb", "leveldb",
@ -370,65 +309,53 @@
"leveldb/opt", "leveldb/opt",
"leveldb/storage", "leveldb/storage",
"leveldb/table", "leveldb/table",
"leveldb/util", "leveldb/util"
] ]
pruneopts = ""
revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd" revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
[[projects]] [[projects]]
digest = "1:cae65c1f3471b1f456a9d8a160440f4824ad50c718a15da4144490fbc3b12e49"
name = "github.com/tinylib/msgp" name = "github.com/tinylib/msgp"
packages = ["msgp"] packages = ["msgp"]
pruneopts = ""
revision = "b2b6a672cf1e5b90748f79b8b81fc8c5cf0571a1" revision = "b2b6a672cf1e5b90748f79b8b81fc8c5cf0571a1"
version = "1.0.2" version = "1.0.2"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:61a86f0be8b466d6e3fbdabb155aaa4006137cb5e3fd3b949329d103fa0ceb0f"
name = "golang.org/x/crypto" name = "golang.org/x/crypto"
packages = [ packages = [
"pbkdf2", "pbkdf2",
"scrypt", "scrypt"
] ]
pruneopts = ""
revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" revision = "0e37d006457bf46f9e6692014ba72ef82c33022c"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
name = "golang.org/x/net" name = "golang.org/x/net"
packages = [ packages = [
"context", "context",
"html", "html",
"html/atom", "html/atom",
"html/charset", "html/charset",
"websocket", "websocket"
] ]
pruneopts = ""
revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
name = "golang.org/x/sync" name = "golang.org/x/sync"
packages = ["errgroup"] packages = ["errgroup"]
pruneopts = ""
revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
name = "golang.org/x/sys" name = "golang.org/x/sys"
packages = [ packages = [
"unix", "unix",
"windows", "windows"
] ]
pruneopts = ""
revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427" revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
[[projects]] [[projects]]
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
name = "golang.org/x/text" name = "golang.org/x/text"
packages = [ packages = [
"encoding", "encoding",
@ -450,108 +377,50 @@
"runes", "runes",
"transform", "transform",
"unicode/cldr", "unicode/cldr",
"unicode/norm", "unicode/norm"
] ]
pruneopts = ""
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0" version = "v0.3.0"
[[projects]] [[projects]]
branch = "master"
digest = "1:88401f863a34cfd82173ef2ad015652c2c47c866affc424bfdd4d5fafad1dd53"
name = "golang.org/x/tools"
packages = [
"go/ast/astutil",
"imports",
"internal/fastwalk",
]
pruneopts = ""
revision = "90fa682c2a6e6a37b3a1364ce2fe1d5e41af9d6d"
[[projects]]
digest = "1:be560af8c72c59788a6748eae60b8e1fe136fefdf049b7dfb6f5b591ea337984"
name = "gopkg.in/DataDog/dd-trace-go.v1" name = "gopkg.in/DataDog/dd-trace-go.v1"
packages = [ packages = [
"ddtrace", "ddtrace",
"ddtrace/ext", "ddtrace/ext",
"ddtrace/internal", "ddtrace/internal",
"ddtrace/tracer", "ddtrace/tracer"
] ]
pruneopts = ""
revision = "bcd20367df871708a36549e7fe36183ee5b4fc55" revision = "bcd20367df871708a36549e7fe36183ee5b4fc55"
version = "v1.3.0" version = "v1.3.0"
[[projects]] [[projects]]
digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
name = "gopkg.in/fsnotify.v1" name = "gopkg.in/fsnotify.v1"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
source = "gopkg.in/fsnotify/fsnotify.v1" source = "gopkg.in/fsnotify/fsnotify.v1"
version = "v1.4.7" version = "v1.4.7"
[[projects]] [[projects]]
branch = "v2" branch = "v2"
digest = "1:a585c075875ab9c344f7840a927f09f3285563f7318e761a1d61d642316f2217"
name = "gopkg.in/karalabe/cookiejar.v2"
packages = ["collections/prque"]
pruneopts = ""
revision = "8dcd6a7f4951f6ff3ee9cbb919a06d8925822e57"
[[projects]]
branch = "v2"
digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
name = "gopkg.in/natefinch/npipe.v2" name = "gopkg.in/natefinch/npipe.v2"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6" revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
[[projects]] [[projects]]
branch = "v1" branch = "v1"
digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
name = "gopkg.in/tomb.v1" name = "gopkg.in/tomb.v1"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8" revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
[[projects]] [[projects]]
digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
name = "gopkg.in/yaml.v2" name = "gopkg.in/yaml.v2"
packages = ["."] packages = ["."]
pruneopts = ""
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1" version = "v2.2.1"
[solve-meta] [solve-meta]
analyzer-name = "dep" analyzer-name = "dep"
analyzer-version = 1 analyzer-version = 1
input-imports = [ inputs-digest = "2fbede45ac5daf4cab0ad947ea580193383fddfe0b5d1ab924bd61959839ba1c"
"github.com/ethereum/go-ethereum",
"github.com/ethereum/go-ethereum/accounts/abi",
"github.com/ethereum/go-ethereum/accounts/abi/bind",
"github.com/ethereum/go-ethereum/common",
"github.com/ethereum/go-ethereum/common/hexutil",
"github.com/ethereum/go-ethereum/core/rawdb",
"github.com/ethereum/go-ethereum/core/types",
"github.com/ethereum/go-ethereum/crypto",
"github.com/ethereum/go-ethereum/ethclient",
"github.com/ethereum/go-ethereum/ethdb",
"github.com/ethereum/go-ethereum/p2p",
"github.com/ethereum/go-ethereum/p2p/discover",
"github.com/ethereum/go-ethereum/params",
"github.com/ethereum/go-ethereum/rlp",
"github.com/ethereum/go-ethereum/rpc",
"github.com/jmoiron/sqlx",
"github.com/lib/pq",
"github.com/mitchellh/go-homedir",
"github.com/onsi/ginkgo",
"github.com/onsi/gomega",
"github.com/onsi/gomega/ghttp",
"github.com/spf13/cobra",
"github.com/spf13/viper",
"golang.org/x/net/context",
"golang.org/x/sync/errgroup",
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer",
]
solver-name = "gps-cdcl" solver-name = "gps-cdcl"
solver-version = 1 solver-version = 1

View File

@ -47,4 +47,4 @@
[[constraint]] [[constraint]]
name = "github.com/ethereum/go-ethereum" name = "github.com/ethereum/go-ethereum"
version = "1.8.15" version = "1.8.18"

View File

@ -2,7 +2,7 @@
[![Join the chat at https://gitter.im/vulcanizeio/VulcanizeDB](https://badges.gitter.im/vulcanizeio/VulcanizeDB.svg)](https://gitter.im/vulcanizeio/VulcanizeDB?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Join the chat at https://gitter.im/vulcanizeio/VulcanizeDB](https://badges.gitter.im/vulcanizeio/VulcanizeDB.svg)](https://gitter.im/vulcanizeio/VulcanizeDB?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Build Status](https://travis-ci.org/vulcanize/vulcanizedb.svg?branch=master)](https://travis-ci.org/vulcanize/vulcanizedb) [![Build Status](https://travis-ci.com/8thlight/maker-vulcanizedb.svg?branch=staging)](https://travis-ci.com/8thlight/maker-vulcanizedb)
## About ## About

View File

@ -2,7 +2,7 @@ package crypto
import ( import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/discv5"
) )
type PublicKeyParser interface { type PublicKeyParser interface {
@ -16,6 +16,6 @@ func (EthPublicKeyParser) ParsePublicKey(privateKey string) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
pubKey := discover.PubkeyID(&np.PublicKey) pubKey := discv5.PubkeyID(&np.PublicKey)
return pubKey.String(), nil return pubKey.String(), nil
} }

View File

@ -42,12 +42,6 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
return fmt.Errorf("model of type %T, not %T", model, BiteModel{}) return fmt.Errorf("model of type %T, not %T", model, BiteModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, biteModel.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log) `INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`, VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`,

View File

@ -39,12 +39,6 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte
return fmt.Errorf("model of type %T, not %T", model, CatFileChopLumpModel{}) return fmt.Errorf("model of type %T, not %T", model, CatFileChopLumpModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, chopLump.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log) `INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -38,12 +38,6 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac
return fmt.Errorf("model of type %T, not %T", model, CatFileFlipModel{}) return fmt.Errorf("model of type %T, not %T", model, CatFileFlipModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, flip.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = repository.db.Exec( _, err = repository.db.Exec(
`INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log) `INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7)`, VALUES($1, $2, $3, $4, $5, $6, $7)`,

View File

@ -38,12 +38,6 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf
return fmt.Errorf("model of type %T, not %T", model, CatFilePitVowModel{}) return fmt.Errorf("model of type %T, not %T", model, CatFilePitVowModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vow.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = repository.db.Exec( _, err = repository.db.Exec(
`INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log) `INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)`, VALUES($1, $2, $3, $4, $5, $6)`,

View File

@ -39,12 +39,6 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er
return fmt.Errorf("model of type %T, not %T", model, DealModel{}) return fmt.Errorf("model of type %T, not %T", model, DealModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, dealModel.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log) `INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)`, VALUES($1, $2, $3, $4, $5, $6)`,

View File

@ -44,12 +44,6 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er
return fmt.Errorf("model of type %T, not %T", model, DentModel{}) return fmt.Errorf("model of type %T, not %T", model, DentModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, dent.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) `INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`, VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`,

View File

@ -38,12 +38,6 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{}
return fmt.Errorf("model of type %T, not %T", model, DripDripModel{}) return fmt.Errorf("model of type %T, not %T", model, DripDripModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, dripDrip.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log) `INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`, VALUES($1, $2, $3, $4, $5)`,

View File

@ -39,12 +39,6 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac
return fmt.Errorf("model of type %T, not %T", model, DripFileIlkModel{}) return fmt.Errorf("model of type %T, not %T", model, DripFileIlkModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, ilk.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log) `INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -39,12 +39,6 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa
return fmt.Errorf("model of type %T, not %T", model, DripFileRepoModel{}) return fmt.Errorf("model of type %T, not %T", model, DripFileRepoModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, repo.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`,

View File

@ -39,12 +39,6 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac
return fmt.Errorf("model of type %T, not %T", model, DripFileVowModel{}) return fmt.Errorf("model of type %T, not %T", model, DripFileVowModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vow.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6)`, VALUES($1, $2, $3, $4, $5, $6)`,

View File

@ -52,7 +52,7 @@ func (transformer LogNoteTransformer) Execute() error {
log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders)) log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
for _, header := range missingHeaders { for _, header := range missingHeaders {
// Fetch the missing logs for a given header // Fetch the missing logs for a given header
matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header.BlockNumber) matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header)
if err != nil { if err != nil {
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
return err return err

View File

@ -48,7 +48,7 @@ func (transformer Transformer) Execute() error {
} }
log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders)) log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
for _, header := range missingHeaders { for _, header := range missingHeaders {
logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber) logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header)
if err != nil { if err != nil {
log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err) log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
return err return err

View File

@ -37,12 +37,6 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{
return fmt.Errorf("model of type %T, not %T", model, FlapKickModel{}) return fmt.Errorf("model of type %T, not %T", model, FlapKickModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, flapKickModel.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) `INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`,

View File

@ -38,12 +38,6 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{}
return fmt.Errorf("model of type %T, not %T", model, FlipKickModel{}) return fmt.Errorf("model of type %T, not %T", model, FlipKickModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, flipKickModel.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log) `INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)`,

View File

@ -38,12 +38,6 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{}
return fmt.Errorf("model of type %T, not %T", flopKick, Model{}) return fmt.Errorf("model of type %T, not %T", flopKick, Model{})
} }
err = shared.ValidateHeaderConsistency(headerID, flopKickModel.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log) `INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`,

View File

@ -39,12 +39,6 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er
return fmt.Errorf("model of type %T, not %T", model, FrobModel{}) return fmt.Errorf("model of type %T, not %T", model, FrobModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, frobModel.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx) _, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx)
VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`, VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`,
headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex) headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex)

View File

@ -39,7 +39,7 @@ var _ = Describe("VowFlog LogNoteTransformer", func() {
db := test_config.NewTestDB(blockchain.Node()) db := test_config.NewTestDB(blockchain.Node())
test_config.CleanTestDB(db) test_config.CleanTestDB(db)
err = persistHeader(db, blockNumber) err = persistHeader(db, blockNumber, blockchain)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(1).To(Equal(1)) Expect(1).To(Equal(1))

View File

@ -39,12 +39,6 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i
return fmt.Errorf("model of type %T, not %T", model, PitFileDebtCeilingModel{}) return fmt.Errorf("model of type %T, not %T", model, PitFileDebtCeilingModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, pitFileDC.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`, VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`,

View File

@ -39,12 +39,6 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface
return fmt.Errorf("model of type %T, not %T", model, PitFileIlkModel{}) return fmt.Errorf("model of type %T, not %T", model, PitFileIlkModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, pitFileIlk.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log) `INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -38,12 +38,6 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{
return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{}) return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, priceUpdate.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log) _, err = tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log)
VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw) VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw)
if err != nil { if err != nil {

View File

@ -1,34 +1,34 @@
package constants package constants
var ( var (
BiteLabel = "bite" BiteLabel = "bite"
CatFileChopLumpLabel = "catFileChopLump" CatFileChopLumpLabel = "catFileChopLump"
CatFileFlipLabel = "catFileFlip" CatFileFlipLabel = "catFileFlip"
CatFilePitVowLabel = "catFilePitVow" CatFilePitVowLabel = "catFilePitVow"
DealLabel = "deal" DealLabel = "deal"
DentLabel = "dent" DentLabel = "dent"
DripDripLabel = "dripDrip" DripDripLabel = "dripDrip"
DripFileIlkLabel = "dripFileIlk" DripFileIlkLabel = "dripFileIlk"
DripFileRepoLabel = "dripFileRepo" DripFileRepoLabel = "dripFileRepo"
DripFileVowLabel = "dripFileVow" DripFileVowLabel = "dripFileVow"
FlapKickLabel = "flapKick" FlapKickLabel = "flapKick"
FlipKickLabel = "flipKick" FlipKickLabel = "flipKick"
FlopKickLabel = "flopKick" FlopKickLabel = "flopKick"
FrobLabel = "frob" FrobLabel = "frob"
PitFileDebtCeilingLabel = "pitFileDebtCeiling" PitFileDebtCeilingLabel = "pitFileDebtCeiling"
PitFileIlkLabel = "pitFileIlk" PitFileIlkLabel = "pitFileIlk"
PriceFeedLabel = "priceFeed" PriceFeedLabel = "priceFeed"
TendLabel = "tend" TendLabel = "tend"
VatFluxLabel = "vatFlux" VatFluxLabel = "vatFlux"
VatFoldLabel = "vatFold" VatFoldLabel = "vatFold"
VatGrabLabel = "vatGrab" VatGrabLabel = "vatGrab"
VatHealLabel = "vatHeal" VatHealLabel = "vatHeal"
VatInitLabel = "vatInit" VatInitLabel = "vatInit"
VatMoveLabel = "vatMove" VatMoveLabel = "vatMove"
VatSlipLabel = "vatSlip" VatSlipLabel = "vatSlip"
VatTollLabel = "vatToll" VatTollLabel = "vatToll"
VatTuneLabel = "vatTune" VatTuneLabel = "vatTune"
VowFlogLabel = "vowFlog" VowFlogLabel = "vowFlog"
) )
func AllTransformerLabels() []string { func AllTransformerLabels() []string {

View File

@ -15,8 +15,6 @@
package shared package shared
import ( import (
"math/big"
"github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
@ -25,7 +23,7 @@ import (
) )
type LogFetcher interface { type LogFetcher interface {
FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error)
} }
type SettableLogFetcher interface { type SettableLogFetcher interface {
@ -47,12 +45,11 @@ func NewFetcher(blockchain core.BlockChain) Fetcher {
} }
} }
func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) { func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
block := big.NewInt(blockNumber)
addresses := hexStringsToAddresses(contractAddresses) addresses := hexStringsToAddresses(contractAddresses)
blockHash := common.HexToHash(header.Hash)
query := ethereum.FilterQuery{ query := ethereum.FilterQuery{
FromBlock: block, BlockHash: &blockHash,
ToBlock: block,
Addresses: addresses, Addresses: addresses,
Topics: topics, Topics: topics,
} }

View File

@ -15,13 +15,12 @@
package shared_test package shared_test
import ( import (
"math/big"
"github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
) )
@ -31,18 +30,20 @@ var _ = Describe("Fetcher", func() {
It("fetches logs based on the given query", func() { It("fetches logs based on the given query", func() {
blockChain := fakes.NewMockBlockChain() blockChain := fakes.NewMockBlockChain()
fetcher := shared.NewFetcher(blockChain) fetcher := shared.NewFetcher(blockChain)
blockNumber := int64(123) header := fakes.FakeHeader
addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"} addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"}
topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}} topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
_, err := fetcher.FetchLogs(addresses, topicZeros, blockNumber) _, err := fetcher.FetchLogs(addresses, topicZeros, header)
address1 := common.HexToAddress("0xfakeAddress") address1 := common.HexToAddress("0xfakeAddress")
address2 := common.HexToAddress("0xanotherFakeAddress") address2 := common.HexToAddress("0xanotherFakeAddress")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
blockHash := common.HexToHash(header.Hash)
expectedQuery := ethereum.FilterQuery{ expectedQuery := ethereum.FilterQuery{
FromBlock: big.NewInt(blockNumber), BlockHash: &blockHash,
ToBlock: big.NewInt(blockNumber),
Addresses: []common.Address{address1, address2}, Addresses: []common.Address{address1, address2},
Topics: topicZeros, Topics: topicZeros,
} }
@ -54,7 +55,7 @@ var _ = Describe("Fetcher", func() {
blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError) blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
fetcher := shared.NewFetcher(blockChain) fetcher := shared.NewFetcher(blockChain)
_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, int64(1)) _, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{})
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(fakes.FakeError)) Expect(err).To(MatchError(fakes.FakeError))

View File

@ -2,41 +2,10 @@ package shared
import ( import (
"database/sql" "database/sql"
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/core/types"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
) )
var ErrHeaderMismatch = errors.New("log's block hash does not match associated header's block hash")
func ValidateHeaderConsistency(headerID int64, rawLog []byte, db *postgres.DB) error {
var log types.Log
err := json.Unmarshal(rawLog, &log)
if err != nil {
return err
}
var hash string
err = db.Get(&hash, `SELECT hash FROM public.headers WHERE id = $1`, headerID)
if err != nil {
return err
}
if hash != log.BlockHash.String() {
err = deleteHeader(headerID, db)
if err != nil {
return err
}
return ErrHeaderMismatch
}
return nil
}
func deleteHeader(headerID int64, db *postgres.DB) error {
_, err := db.Exec(`DELETE FROM public.headers WHERE id = $1`, headerID)
return err
}
func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error { func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`) _, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
VALUES ($1, $2) VALUES ($1, $2)
@ -59,14 +28,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
var err error var err error
if endingBlockNumber == -1 { if endingBlockNumber == -1 {
query = `SELECT headers.id, headers.block_number FROM headers query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
AND headers.block_number >= $1 AND headers.block_number >= $1
AND headers.eth_node_fingerprint = $2` AND headers.eth_node_fingerprint = $2`
err = db.Select(&result, query, startingBlockNumber, db.Node.ID) err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
} else { } else {
query = `SELECT headers.id, headers.block_number FROM headers query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
LEFT JOIN checked_headers on headers.id = header_id LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE) WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
AND headers.block_number >= $1 AND headers.block_number >= $1

View File

@ -44,12 +44,6 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er
return fmt.Errorf("model of type %T, not %T", model, TendModel{}) return fmt.Errorf("model of type %T, not %T", model, TendModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, tend.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log) `INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`, VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`,

View File

@ -29,10 +29,10 @@ type MockLogFetcher struct {
SetBcCalled bool SetBcCalled bool
} }
func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) { func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses) mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses)
mlf.FetchedTopics = topics mlf.FetchedTopics = topics
mlf.FetchedBlocks = append(mlf.FetchedBlocks, blockNumber) mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber)
return mlf.FetchedLogs, mlf.fetcherError return mlf.FetchedLogs, mlf.fetcherError
} }

View File

@ -15,7 +15,6 @@
package shared_behaviors package shared_behaviors
import ( import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
@ -24,7 +23,6 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
"github.com/vulcanize/vulcanizedb/pkg/fakes" "github.com/vulcanize/vulcanizedb/pkg/fakes"
"github.com/vulcanize/vulcanizedb/pkg/transformers/factories" "github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data" "github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
"github.com/vulcanize/vulcanizedb/test_config" "github.com/vulcanize/vulcanizedb/test_config"
"math/rand" "math/rand"
@ -103,35 +101,6 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint")) Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
}) })
Describe("when log's block hash does not match associated header's block hash", func() {
var (
differentHeaderID int64
err error
)
BeforeEach(func() {
differentHeader := fakes.FakeHeader
differentHeader.BlockNumber = fakes.FakeHeader.BlockNumber + 1
differentHeader.Hash = common.BytesToHash(append(fakes.FakeHash.Bytes(), 1)).String()
differentHeaderID, err = headerRepository.CreateOrUpdateHeader(differentHeader)
Expect(err).NotTo(HaveOccurred())
err = repository.Create(differentHeaderID, []interface{}{logEventModel})
})
It("returns error", func() {
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrHeaderMismatch))
})
It("removes header", func() {
var headerIDs []int64
err = db.Select(&headerIDs, `SELECT id FROM public.headers`)
Expect(err).NotTo(HaveOccurred())
Expect(headerIDs).NotTo(ContainElement(differentHeaderID))
})
})
It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() { It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() {
err = repository.Create(headerID, []interface{}{logEventModel}) err = repository.Create(headerID, []interface{}{logEventModel})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())

View File

@ -39,12 +39,6 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatFluxModel{}) return fmt.Errorf("model of type %T, not %T", model, VatFluxModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatFlux.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log) _, err = tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)`, VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)`,
headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw) headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw)

View File

@ -39,12 +39,6 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatFoldModel{}) return fmt.Errorf("model of type %T, not %T", model, VatFoldModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatFold.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log) `INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -25,12 +25,6 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatGrabModel{}) return fmt.Errorf("model of type %T, not %T", model, VatGrabModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatGrab.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log) `INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`,

View File

@ -44,12 +44,6 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatHealModel{}) return fmt.Errorf("model of type %T, not %T", model, VatHealModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatHeal.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log) _, err := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,
headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw) headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw)

View File

@ -39,12 +39,6 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatInitModel{}) return fmt.Errorf("model of type %T, not %T", model, VatInitModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatInit.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log) `INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log)
VALUES($1, $2, $3, $4, $5)`, VALUES($1, $2, $3, $4, $5)`,

View File

@ -39,12 +39,6 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatMoveModel{}) return fmt.Errorf("model of type %T, not %T", model, VatMoveModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatMove.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log) `INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log)
VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -24,12 +24,6 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatSlipModel{}) return fmt.Errorf("model of type %T, not %T", model, VatSlipModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatSlip.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log) `INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -24,12 +24,6 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatTollModel{}) return fmt.Errorf("model of type %T, not %T", model, VatTollModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatToll.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, log_idx, raw_log) `INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

View File

@ -5,6 +5,7 @@ import (
"github.com/vulcanize/vulcanizedb/pkg/core" "github.com/vulcanize/vulcanizedb/pkg/core"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres" "github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared" "github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
) )
type VatTuneRepository struct { type VatTuneRepository struct {
@ -23,12 +24,6 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VatTuneModel{}) return fmt.Errorf("model of type %T, not %T", model, VatTuneModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, vatTune.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log) `INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log)
VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`, VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`,
@ -40,10 +35,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
} }
} }
_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked) err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTuneChecked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET vat_tune_checked = $2`, headerID, true)
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
return err return err
@ -52,28 +44,11 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
} }
func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error { func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error {
_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked) return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked)
VALUES ($1, $2)
ON CONFLICT (header_id) DO
UPDATE SET vat_tune_checked = $2`, headerID, true)
return err
} }
func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) { func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
var result []core.Header return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked)
err := repository.db.Select(
&result,
`SELECT headers.id, headers.block_number FROM headers
LEFT JOIN checked_headers on headers.id = header_id
WHERE (header_id ISNULL OR vat_tune_checked IS FALSE)
AND headers.block_number >= $1
AND headers.block_number <= $2
AND headers.eth_node_fingerprint = $3`,
startingBlockNumber,
endingBlockNumber,
repository.db.Node.ID,
)
return result, err
} }
func (repository *VatTuneRepository) SetDB(db *postgres.DB) { func (repository *VatTuneRepository) SetDB(db *postgres.DB) {

View File

@ -39,12 +39,6 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{})
return fmt.Errorf("model of type %T, not %T", model, VowFlogModel{}) return fmt.Errorf("model of type %T, not %T", model, VowFlogModel{})
} }
err = shared.ValidateHeaderConsistency(headerID, flog.Raw, repository.db)
if err != nil {
tx.Rollback()
return err
}
_, err = tx.Exec( _, err = tx.Exec(
`INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log) `INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log)
VALUES($1, $2::NUMERIC, $3, $4, $5)`, VALUES($1, $2::NUMERIC, $3, $4, $5)`,

View File

@ -9,24 +9,27 @@ les/ @zsfelfoldi
light/ @zsfelfoldi light/ @zsfelfoldi
mobile/ @karalabe mobile/ @karalabe
p2p/ @fjl @zsfelfoldi p2p/ @fjl @zsfelfoldi
p2p/simulations @lmars
p2p/protocols @zelig
swarm/api/http @justelad
swarm/bmt @zelig swarm/bmt @zelig
swarm/dev @lmars swarm/dev @lmars
swarm/fuse @jmozah @holisticode swarm/fuse @jmozah @holisticode
swarm/grafana_dashboards @nonsense swarm/grafana_dashboards @nonsense
swarm/metrics @nonsense @holisticode swarm/metrics @nonsense @holisticode
swarm/multihash @nolash swarm/multihash @nolash
swarm/network/bitvector @zelig @janos @gbalint swarm/network/bitvector @zelig @janos
swarm/network/priorityqueue @zelig @janos @gbalint swarm/network/priorityqueue @zelig @janos
swarm/network/simulations @zelig swarm/network/simulations @zelig @janos
swarm/network/stream @janos @zelig @gbalint @holisticode @justelad swarm/network/stream @janos @zelig @holisticode @justelad
swarm/network/stream/intervals @janos swarm/network/stream/intervals @janos
swarm/network/stream/testing @zelig swarm/network/stream/testing @zelig
swarm/pot @zelig swarm/pot @zelig
swarm/pss @nolash @zelig @nonsense swarm/pss @nolash @zelig @nonsense
swarm/services @zelig swarm/services @zelig
swarm/state @justelad swarm/state @justelad
swarm/storage/encryption @gbalint @zelig @nagydani swarm/storage/encryption @zelig @nagydani
swarm/storage/mock @janos swarm/storage/mock @janos
swarm/storage/mru @nolash swarm/storage/feed @nolash @jpeletier
swarm/testutil @lmars swarm/testutil @lmars
whisper/ @gballet @gluk256 whisper/ @gballet @gluk256

View File

@ -1,16 +1,40 @@
# Contributing
Thank you for considering to help out with the source code! We welcome
contributions from anyone on the internet, and are grateful for even the
smallest of fixes!
If you'd like to contribute to go-ethereum, please fork, fix, commit and send a
pull request for the maintainers to review and merge into the main code base. If
you wish to submit more complex changes though, please check up with the core
devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to
ensure those changes are in line with the general philosophy of the project
and/or get some early feedback which can make both your efforts much lighter as
well as our review and merge procedures quick and simple.
## Coding guidelines
Please make sure your contributions adhere to our coding guidelines:
* Code must adhere to the official Go
[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines
(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
* Code must be documented adhering to the official Go
[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
* Pull requests need to be based on and opened against the `master` branch.
* Commit messages should be prefixed with the package(s) they modify.
* E.g. "eth, rpc: make trace configs optional"
## Can I have feature X ## Can I have feature X
Before you do a feature request please check and make sure that it isn't possible Before you submit a feature request, please check and make sure that it isn't
through some other means. The JavaScript enabled console is a powerful feature possible through some other means. The JavaScript-enabled console is a powerful
in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info feature in the right hands. Please check our
[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
and help. and help.
## Contributing ## Configuration, dependencies, and tests
If you'd like to contribute to go-ethereum please fork, fix, commit and Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
send a pull request. Commits which do not comply with the coding standards for more details on configuring your environment, managing project dependencies
are ignored (use gofmt!). and testing procedures.
See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
for more details on configuring your environment, testing, and
dependency management.

View File

@ -7,5 +7,5 @@ closeComment: >
This issue has been automatically closed because there has been no response This issue has been automatically closed because there has been no response
to our request for more information from the original author. With only the to our request for more information from the original author. With only the
information that is currently in the issue, we don't have enough information information that is currently in the issue, we don't have enough information
to take action. Please reach out if you have or find the answers we need so to take action. Please reach out if you have more relevant information or
that we can investigate further. answers to our questions so that we can investigate further.

View File

@ -6,19 +6,19 @@ matrix:
- os: linux - os: linux
dist: trusty dist: trusty
sudo: required sudo: required
go: 1.9.x go: 1.10.x
script: script:
- sudo modprobe fuse - sudo modprobe fuse
- sudo chmod 666 /dev/fuse - sudo chmod 666 /dev/fuse
- sudo chown root:$USER /etc/fuse.conf - sudo chown root:$USER /etc/fuse.conf
- go run build/ci.go install - go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES - go run build/ci.go test -coverage $TEST_PACKAGES
# These are the latest Go versions. # These are the latest Go versions.
- os: linux - os: linux
dist: trusty dist: trusty
sudo: required sudo: required
go: 1.10.x go: 1.11.x
script: script:
- sudo modprobe fuse - sudo modprobe fuse
- sudo chmod 666 /dev/fuse - sudo chmod 666 /dev/fuse
@ -27,7 +27,7 @@ matrix:
- go run build/ci.go test -coverage $TEST_PACKAGES - go run build/ci.go test -coverage $TEST_PACKAGES
- os: osx - os: osx
go: 1.10.x go: 1.11.x
script: script:
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- go run build/ci.go install - go run build/ci.go install
@ -36,7 +36,7 @@ matrix:
# This builder only tests code linters on latest version of Go # This builder only tests code linters on latest version of Go
- os: linux - os: linux
dist: trusty dist: trusty
go: 1.10.x go: 1.11.x
env: env:
- lint - lint
git: git:
@ -45,9 +45,10 @@ matrix:
- go run build/ci.go lint - go run build/ci.go lint
# This builder does the Ubuntu PPA upload # This builder does the Ubuntu PPA upload
- os: linux - if: type = push
os: linux
dist: trusty dist: trusty
go: 1.10.x go: 1.11.x
env: env:
- ubuntu-ppa - ubuntu-ppa
git: git:
@ -63,10 +64,11 @@ matrix:
- go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
# This builder does the Linux Azure uploads # This builder does the Linux Azure uploads
- os: linux - if: type = push
os: linux
dist: trusty dist: trusty
sudo: required sudo: required
go: 1.10.x go: 1.11.x
env: env:
- azure-linux - azure-linux
git: git:
@ -96,11 +98,12 @@ matrix:
- go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
# This builder does the Linux Azure MIPS xgo uploads # This builder does the Linux Azure MIPS xgo uploads
- os: linux - if: type = push
os: linux
dist: trusty dist: trusty
services: services:
- docker - docker
go: 1.10.x go: 1.11.x
env: env:
- azure-linux-mips - azure-linux-mips
git: git:
@ -123,7 +126,8 @@ matrix:
- go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
# This builder does the Android Maven and Azure uploads # This builder does the Android Maven and Azure uploads
- os: linux - if: type = push
os: linux
dist: trusty dist: trusty
addons: addons:
apt: apt:
@ -144,7 +148,7 @@ matrix:
git: git:
submodules: false # avoid cloning ethereum/tests submodules: false # avoid cloning ethereum/tests
before_install: before_install:
- curl https://storage.googleapis.com/golang/go1.10.3.linux-amd64.tar.gz | tar -xz - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH - export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go - export GOROOT=`pwd`/go
- export GOPATH=$HOME/go - export GOPATH=$HOME/go
@ -160,8 +164,9 @@ matrix:
- go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
# This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
- os: osx - if: type = push
go: 1.10.x os: osx
go: 1.11.x
env: env:
- azure-osx - azure-osx
- azure-ios - azure-ios
@ -188,19 +193,13 @@ matrix:
- go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
# This builder does the Azure archive purges to avoid accumulating junk # This builder does the Azure archive purges to avoid accumulating junk
- os: linux - if: type = cron
os: linux
dist: trusty dist: trusty
go: 1.10.x go: 1.11.x
env: env:
- azure-purge - azure-purge
git: git:
submodules: false # avoid cloning ethereum/tests submodules: false # avoid cloning ethereum/tests
script: script:
- go run build/ci.go purge -store gethstore/builds -days 14 - go run build/ci.go purge -store gethstore/builds -days 14
notifications:
webhooks:
urls:
- https://webhooks.gitter.im/e/e09ccdce1048c5e03445
on_success: change
on_failure: always

View File

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container # Build Geth in a stock Go builder container
FROM golang:1.10-alpine as builder FROM golang:1.11-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers RUN apk add --no-cache make gcc musl-dev linux-headers

View File

@ -1,5 +1,5 @@
# Build Geth in a stock Go builder container # Build Geth in a stock Go builder container
FROM golang:1.10-alpine as builder FROM golang:1.11-alpine as builder
RUN apk add --no-cache make gcc musl-dev linux-headers RUN apk add --no-cache make gcc musl-dev linux-headers

View File

@ -57,6 +57,9 @@ devtools:
@type "solc" 2> /dev/null || echo 'Please install solc' @type "solc" 2> /dev/null || echo 'Please install solc'
@type "protoc" 2> /dev/null || echo 'Please install protoc' @type "protoc" 2> /dev/null || echo 'Please install protoc'
swarm-devtools:
env GOBIN= go install ./cmd/swarm/mimegen
# Cross Compilation Targets (xgo) # Cross Compilation Targets (xgo)
geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios

View File

@ -7,7 +7,7 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
)](https://godoc.org/github.com/ethereum/go-ethereum) )](https://godoc.org/github.com/ethereum/go-ethereum)
[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
Automated builds are available for stable releases and the unstable master branch. Automated builds are available for stable releases and the unstable master branch.
Binary archives are published at https://geth.ethereum.org/downloads/. Binary archives are published at https://geth.ethereum.org/downloads/.

View File

@ -137,6 +137,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
// MethodById looks up a method by the 4-byte id // MethodById looks up a method by the 4-byte id
// returns nil if none found // returns nil if none found
func (abi *ABI) MethodById(sigdata []byte) (*Method, error) { func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
if len(sigdata) < 4 {
return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
}
for _, method := range abi.Methods { for _, method := range abi.Methods {
if bytes.Equal(method.Id(), sigdata[:4]) { if bytes.Equal(method.Id(), sigdata[:4]) {
return &method, nil return &method, nil

View File

@ -711,5 +711,14 @@ func TestABI_MethodById(t *testing.T) {
t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id())) t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
} }
} }
// Also test empty
if _, err := abi.MethodById([]byte{0x00}); err == nil {
t.Errorf("Expected error, too short to decode data")
}
if _, err := abi.MethodById([]byte{}); err == nil {
t.Errorf("Expected error, too short to decode data")
}
if _, err := abi.MethodById(nil); err == nil {
t.Errorf("Expected error, nil is short to decode data")
}
} }

View File

@ -69,7 +69,7 @@ func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBac
database := ethdb.NewMemDatabase() database := ethdb.NewMemDatabase()
genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
genesis.MustCommit(database) genesis.MustCommit(database)
blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
backend := &SimulatedBackend{ backend := &SimulatedBackend{
database: database, database: database,
@ -208,7 +208,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad
} }
// SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
// chain doens't have miners, we just return a gas price of 1 for any call. // chain doesn't have miners, we just return a gas price of 1 for any call.
func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
return big.NewInt(1), nil return big.NewInt(1), nil
} }

View File

@ -23,13 +23,13 @@ package bind
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"go/format"
"regexp" "regexp"
"strings" "strings"
"text/template" "text/template"
"unicode" "unicode"
"github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi"
"golang.org/x/tools/imports"
) )
// Lang is a target programming language selector to generate bindings for. // Lang is a target programming language selector to generate bindings for.
@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
if err := tmpl.Execute(buffer, data); err != nil { if err := tmpl.Execute(buffer, data); err != nil {
return "", err return "", err
} }
// For Go bindings pass the code through goimports to clean it up and double check // For Go bindings pass the code through gofmt to clean it up
if lang == LangGo { if lang == LangGo {
code, err := imports.Process(".", buffer.Bytes(), nil) code, err := format.Source(buffer.Bytes())
if err != nil { if err != nil {
return "", fmt.Errorf("%v\n%s", err, buffer) return "", fmt.Errorf("%v\n%s", err, buffer)
} }

File diff suppressed because one or more lines are too long

View File

@ -64,6 +64,30 @@ const tmplSourceGo = `
package {{.Package}} package {{.Package}}
import (
"math/big"
"strings"
ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
)
// Reference imports to suppress errors if they are not otherwise used.
var (
_ = big.NewInt
_ = strings.NewReader
_ = ethereum.NotFound
_ = abi.U256
_ = bind.Bind
_ = common.Big1
_ = types.BloomLookup
_ = event.NewSubscription
)
{{range $contract := .Contracts}} {{range $contract := .Contracts}}
// {{.Type}}ABI is the input ABI used to generate the binding from. // {{.Type}}ABI is the input ABI used to generate the binding from.
const {{.Type}}ABI = "{{.InputABI}}" const {{.Type}}ABI = "{{.InputABI}}"

View File

@ -103,7 +103,12 @@ func NewType(t string) (typ Type, err error) {
return typ, err return typ, err
} }
// parse the type and size of the abi-type. // parse the type and size of the abi-type.
parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0] matches := typeRegex.FindAllStringSubmatch(t, -1)
if len(matches) == 0 {
return Type{}, fmt.Errorf("invalid type '%v'", t)
}
parsedType := matches[0]
// varSize is the size of the variable // varSize is the size of the variable
var varSize int var varSize int
if len(parsedType[3]) > 0 { if len(parsedType[3]) > 0 {

View File

@ -25,8 +25,17 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
) )
var (
maxUint256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
big.NewInt(-1))
maxInt256 = big.NewInt(0).Add(
big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
big.NewInt(-1))
)
// reads the integer based on its kind // reads the integer based on its kind
func readInteger(kind reflect.Kind, b []byte) interface{} { func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
switch kind { switch kind {
case reflect.Uint8: case reflect.Uint8:
return b[len(b)-1] return b[len(b)-1]
@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
case reflect.Int64: case reflect.Int64:
return int64(binary.BigEndian.Uint64(b[len(b)-8:])) return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
default: default:
return new(big.Int).SetBytes(b) // the only case lefts for integer is int256/uint256.
// big.SetBytes can't tell if a number is negative, positive on itself.
// On EVM, if the returned number > max int256, it is negative.
ret := new(big.Int).SetBytes(b)
if typ == UintTy {
return ret
}
if ret.Cmp(maxInt256) > 0 {
ret.Add(maxUint256, big.NewInt(0).Neg(ret))
ret.Add(ret, big.NewInt(1))
ret.Neg(ret)
}
return ret
} }
} }
@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
case StringTy: // variable arrays are written at the end of the return bytes case StringTy: // variable arrays are written at the end of the return bytes
return string(output[begin : begin+end]), nil return string(output[begin : begin+end]), nil
case IntTy, UintTy: case IntTy, UintTy:
return readInteger(t.Kind, returnOutput), nil return readInteger(t.T, t.Kind, returnOutput), nil
case BoolTy: case BoolTy:
return readBool(returnOutput) return readBool(returnOutput)
case AddressTy: case AddressTy:

View File

@ -117,6 +117,11 @@ var unpackTests = []unpackTest{
enc: "0000000000000000000000000000000000000000000000000000000000000001", enc: "0000000000000000000000000000000000000000000000000000000000000001",
want: big.NewInt(1), want: big.NewInt(1),
}, },
{
def: `[{"type": "int256"}]`,
enc: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
want: big.NewInt(-1),
},
{ {
def: `[{"type": "address"}]`, def: `[{"type": "address"}]`,
enc: "0000000000000000000000000100000000000000000000000000000000000000", enc: "0000000000000000000000000100000000000000000000000000000000000000",

View File

@ -30,8 +30,8 @@ import (
var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0} var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
// DefaultBaseDerivationPath is the base path from which custom derivation endpoints // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second // are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
// at m/44'/60'/0'/1, etc. // at m/44'/60'/0'/0/1, etc.
var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0} var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
// DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

View File

@ -66,19 +66,19 @@ type plainKeyJSON struct {
type encryptedKeyJSONV3 struct { type encryptedKeyJSONV3 struct {
Address string `json:"address"` Address string `json:"address"`
Crypto cryptoJSON `json:"crypto"` Crypto CryptoJSON `json:"crypto"`
Id string `json:"id"` Id string `json:"id"`
Version int `json:"version"` Version int `json:"version"`
} }
type encryptedKeyJSONV1 struct { type encryptedKeyJSONV1 struct {
Address string `json:"address"` Address string `json:"address"`
Crypto cryptoJSON `json:"crypto"` Crypto CryptoJSON `json:"crypto"`
Id string `json:"id"` Id string `json:"id"`
Version string `json:"version"` Version string `json:"version"`
} }
type cryptoJSON struct { type CryptoJSON struct {
Cipher string `json:"cipher"` Cipher string `json:"cipher"`
CipherText string `json:"ciphertext"` CipherText string `json:"ciphertext"`
CipherParams cipherparamsJSON `json:"cipherparams"` CipherParams cipherparamsJSON `json:"cipherparams"`
@ -179,26 +179,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
return key, a, err return key, a, err
} }
func writeKeyFile(file string, content []byte) error { func writeTemporaryKeyFile(file string, content []byte) (string, error) {
// Create the keystore directory with appropriate permissions // Create the keystore directory with appropriate permissions
// in case it is not present yet. // in case it is not present yet.
const dirPerm = 0700 const dirPerm = 0700
if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil { if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
return err return "", err
} }
// Atomic write: create a temporary hidden file first // Atomic write: create a temporary hidden file first
// then move it into place. TempFile assigns mode 0600. // then move it into place. TempFile assigns mode 0600.
f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp") f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
if err != nil { if err != nil {
return err return "", err
} }
if _, err := f.Write(content); err != nil { if _, err := f.Write(content); err != nil {
f.Close() f.Close()
os.Remove(f.Name()) os.Remove(f.Name())
return err return "", err
} }
f.Close() f.Close()
return os.Rename(f.Name(), file) return f.Name(), nil
}
func writeKeyFile(file string, content []byte) error {
name, err := writeTemporaryKeyFile(file, content)
if err != nil {
return err
}
return os.Rename(name, file)
} }
// keyFileName implements the naming convention for keyfiles: // keyFileName implements the naming convention for keyfiles:

View File

@ -78,7 +78,7 @@ type unlocked struct {
// NewKeyStore creates a keystore for the given directory. // NewKeyStore creates a keystore for the given directory.
func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore { func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
keydir, _ = filepath.Abs(keydir) keydir, _ = filepath.Abs(keydir)
ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}} ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
ks.init(keydir) ks.init(keydir)
return ks return ks
} }

View File

@ -35,6 +35,7 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"os"
"path/filepath" "path/filepath"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
@ -72,6 +73,10 @@ type keyStorePassphrase struct {
keysDirPath string keysDirPath string
scryptN int scryptN int
scryptP int scryptP int
// skipKeyFileVerification disables the security-feature which does
// reads and decrypts any newly created keyfiles. This should be 'false' in all
// cases except tests -- setting this to 'true' is not recommended.
skipKeyFileVerification bool
} }
func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) { func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
@ -93,7 +98,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
// StoreKey generates a key, encrypts with 'auth' and stores in the given directory // StoreKey generates a key, encrypts with 'auth' and stores in the given directory
func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) { func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, rand.Reader, auth) _, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
return a.Address, err return a.Address, err
} }
@ -102,7 +107,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
if err != nil { if err != nil {
return err return err
} }
return writeKeyFile(filename, keyjson) // Write into temporary file
tmpName, err := writeTemporaryKeyFile(filename, keyjson)
if err != nil {
return err
}
if !ks.skipKeyFileVerification {
// Verify that we can decrypt the file with the given password.
_, err = ks.GetKey(key.Address, tmpName, auth)
if err != nil {
msg := "An error was encountered when saving and verifying the keystore file. \n" +
"This indicates that the keystore is corrupted. \n" +
"The corrupted file is stored at \n%v\n" +
"Please file a ticket at:\n\n" +
"https://github.com/ethereum/go-ethereum/issues." +
"The error was : %s"
return fmt.Errorf(msg, tmpName, err)
}
}
return os.Rename(tmpName, filename)
} }
func (ks keyStorePassphrase) JoinPath(filename string) string { func (ks keyStorePassphrase) JoinPath(filename string) string {
@ -112,29 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
return filepath.Join(ks.keysDirPath, filename) return filepath.Join(ks.keysDirPath, filename)
} }
// EncryptKey encrypts a key using the specified scrypt parameters into a json // Encryptdata encrypts the data given as 'data' with the password 'auth'.
// blob that can be decrypted later on. func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
authArray := []byte(auth)
salt := make([]byte, 32) salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil { if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error()) panic("reading from crypto/rand failed: " + err.Error())
} }
derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen) derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
if err != nil { if err != nil {
return nil, err return CryptoJSON{}, err
} }
encryptKey := derivedKey[:16] encryptKey := derivedKey[:16]
keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
iv := make([]byte, aes.BlockSize) // 16 iv := make([]byte, aes.BlockSize) // 16
if _, err := io.ReadFull(rand.Reader, iv); err != nil { if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic("reading from crypto/rand failed: " + err.Error()) panic("reading from crypto/rand failed: " + err.Error())
} }
cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv) cipherText, err := aesCTRXOR(encryptKey, data, iv)
if err != nil { if err != nil {
return nil, err return CryptoJSON{}, err
} }
mac := crypto.Keccak256(derivedKey[16:32], cipherText) mac := crypto.Keccak256(derivedKey[16:32], cipherText)
@ -144,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
scryptParamsJSON["p"] = scryptP scryptParamsJSON["p"] = scryptP
scryptParamsJSON["dklen"] = scryptDKLen scryptParamsJSON["dklen"] = scryptDKLen
scryptParamsJSON["salt"] = hex.EncodeToString(salt) scryptParamsJSON["salt"] = hex.EncodeToString(salt)
cipherParamsJSON := cipherparamsJSON{ cipherParamsJSON := cipherparamsJSON{
IV: hex.EncodeToString(iv), IV: hex.EncodeToString(iv),
} }
cryptoStruct := cryptoJSON{ cryptoStruct := CryptoJSON{
Cipher: "aes-128-ctr", Cipher: "aes-128-ctr",
CipherText: hex.EncodeToString(cipherText), CipherText: hex.EncodeToString(cipherText),
CipherParams: cipherParamsJSON, CipherParams: cipherParamsJSON,
@ -157,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
KDFParams: scryptParamsJSON, KDFParams: scryptParamsJSON,
MAC: hex.EncodeToString(mac), MAC: hex.EncodeToString(mac),
} }
return cryptoStruct, nil
}
// EncryptKey encrypts a key using the specified scrypt parameters into a json
// blob that can be decrypted later on.
func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
if err != nil {
return nil, err
}
encryptedKeyJSONV3 := encryptedKeyJSONV3{ encryptedKeyJSONV3 := encryptedKeyJSONV3{
hex.EncodeToString(key.Address[:]), hex.EncodeToString(key.Address[:]),
cryptoStruct, cryptoStruct,
@ -203,43 +233,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
PrivateKey: key, PrivateKey: key,
}, nil }, nil
} }
func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
if cryptoJson.Cipher != "aes-128-ctr" {
return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)
}
mac, err := hex.DecodeString(cryptoJson.MAC)
if err != nil {
return nil, err
}
iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
if err != nil {
return nil, err
}
cipherText, err := hex.DecodeString(cryptoJson.CipherText)
if err != nil {
return nil, err
}
derivedKey, err := getKDFKey(cryptoJson, auth)
if err != nil {
return nil, err
}
calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
if !bytes.Equal(calculatedMAC, mac) {
return nil, ErrDecrypt
}
plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
if err != nil {
return nil, err
}
return plainText, err
}
func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) { func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
if keyProtected.Version != version { if keyProtected.Version != version {
return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version) return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
} }
if keyProtected.Crypto.Cipher != "aes-128-ctr" {
return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
}
keyId = uuid.Parse(keyProtected.Id) keyId = uuid.Parse(keyProtected.Id)
mac, err := hex.DecodeString(keyProtected.Crypto.MAC) plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
if err != nil {
return nil, nil, err
}
iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
if err != nil {
return nil, nil, err
}
cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
if err != nil {
return nil, nil, err
}
derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
if err != nil {
return nil, nil, err
}
calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
if !bytes.Equal(calculatedMAC, mac) {
return nil, nil, ErrDecrypt
}
plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -280,7 +315,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt
return plainText, keyId, err return plainText, keyId, err
} }
func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) { func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
authArray := []byte(auth) authArray := []byte(auth)
salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string)) salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
if err != nil { if err != nil {

View File

@ -37,7 +37,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
t.Fatal(err) t.Fatal(err)
} }
if encrypted { if encrypted {
ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP} ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
} else { } else {
ks = &keyStorePlain{d} ks = &keyStorePlain{d}
} }
@ -191,7 +191,7 @@ func TestV1_1(t *testing.T) {
func TestV1_2(t *testing.T) { func TestV1_2(t *testing.T) {
t.Parallel() t.Parallel()
ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP} ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e") addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e" file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
k, err := ks.GetKey(addr, file, "g") k, err := ks.GetKey(addr, file, "g")

View File

@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
signer = new(types.HomesteadSigner) signer = new(types.HomesteadSigner)
} else { } else {
signer = types.NewEIP155Signer(chainID) signer = types.NewEIP155Signer(chainID)
signature[64] = signature[64] - byte(chainID.Uint64()*2+35) signature[64] -= byte(chainID.Uint64()*2 + 35)
} }
signed, err := tx.WithSignature(signer, signature) signed, err := tx.WithSignature(signer, signature)
if err != nil { if err != nil {

View File

@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
signer = new(types.HomesteadSigner) signer = new(types.HomesteadSigner)
} else { } else {
signer = types.NewEIP155Signer(chainID) signer = types.NewEIP155Signer(chainID)
signature[64] = signature[64] - byte(chainID.Uint64()*2+35) signature[64] -= byte(chainID.Uint64()*2 + 35)
} }
// Inject the final signature into the transaction and sanity check the sender // Inject the final signature into the transaction and sanity check the sender
signed, err := tx.WithSignature(signer, signature) signed, err := tx.WithSignature(signer, signature)

View File

@ -23,8 +23,8 @@ environment:
install: install:
- git submodule update --init - git submodule update --init
- rmdir C:\go /s /q - rmdir C:\go /s /q
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.3.windows-%GETH_ARCH%.zip - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
- 7z x go1.10.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version - go version
- gcc --version - gcc --version

View File

@ -320,9 +320,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
// "tests" also includes static analysis tools such as vet. // "tests" also includes static analysis tools such as vet.
func doTest(cmdline []string) { func doTest(cmdline []string) {
var ( coverage := flag.Bool("coverage", false, "Whether to record code coverage")
coverage = flag.Bool("coverage", false, "Whether to record code coverage")
)
flag.CommandLine.Parse(cmdline) flag.CommandLine.Parse(cmdline)
env := build.Env() env := build.Env()
@ -332,14 +330,11 @@ func doTest(cmdline []string) {
} }
packages = build.ExpandPackagesNoVendor(packages) packages = build.ExpandPackagesNoVendor(packages)
// Run analysis tools before the tests.
build.MustRun(goTool("vet", packages...))
// Run the actual tests. // Run the actual tests.
gotest := goTool("test", buildFlags(env)...)
// Test a single package at a time. CI builders are slow // Test a single package at a time. CI builders are slow
// and some tests run into timeouts under load. // and some tests run into timeouts under load.
gotest.Args = append(gotest.Args, "-p", "1") gotest := goTool("test", buildFlags(env)...)
gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
if *coverage { if *coverage {
gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
} }
@ -1040,7 +1035,7 @@ func xgoTool(args []string) *exec.Cmd {
func doPurge(cmdline []string) { func doPurge(cmdline []string) {
var ( var (
store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`) limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
) )
flag.CommandLine.Parse(cmdline) flag.CommandLine.Parse(cmdline)

View File

@ -75,7 +75,7 @@ func main() {
bins []string bins []string
types []string types []string
) )
if *solFlag != "" || *abiFlag == "-" { if *solFlag != "" || (*abiFlag == "-" && *pkgFlag == "") {
// Generate the list of types to exclude from binding // Generate the list of types to exclude from binding
exclude := make(map[string]bool) exclude := make(map[string]bool)
for _, kind := range strings.Split(*excFlag, ",") { for _, kind := range strings.Split(*excFlag, ",") {
@ -111,7 +111,13 @@ func main() {
} }
} else { } else {
// Otherwise load up the ABI, optional bytecode and type name from the parameters // Otherwise load up the ABI, optional bytecode and type name from the parameters
abi, err := ioutil.ReadFile(*abiFlag) var abi []byte
var err error
if *abiFlag == "-" {
abi, err = ioutil.ReadAll(os.Stdin)
} else {
abi, err = ioutil.ReadFile(*abiFlag)
}
if err != nil { if err != nil {
fmt.Printf("Failed to read input ABI: %v\n", err) fmt.Printf("Failed to read input ABI: %v\n", err)
os.Exit(-1) os.Exit(-1)
@ -155,6 +161,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return compiler.ParseCombinedJSON(bytes, "", "", "", "") return compiler.ParseCombinedJSON(bytes, "", "", "", "")
} }

View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/discv5"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/nat"
"github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/p2p/netutil"
) )
@ -37,7 +38,7 @@ func main() {
var ( var (
listenAddr = flag.String("addr", ":30301", "listen address") listenAddr = flag.String("addr", ":30301", "listen address")
genKey = flag.String("genkey", "", "generate a node key") genKey = flag.String("genkey", "", "generate a node key")
writeAddr = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit") writeAddr = flag.Bool("writeaddress", false, "write out the node's public key and quit")
nodeKeyFile = flag.String("nodekey", "", "private key filename") nodeKeyFile = flag.String("nodekey", "", "private key filename")
nodeKeyHex = flag.String("nodekeyhex", "", "private key as hex (for testing)") nodeKeyHex = flag.String("nodekeyhex", "", "private key as hex (for testing)")
natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)") natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@ -85,7 +86,7 @@ func main() {
} }
if *writeAddr { if *writeAddr {
fmt.Printf("%v\n", discover.PubkeyID(&nodeKey.PublicKey)) fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
os.Exit(0) os.Exit(0)
} }
@ -118,16 +119,17 @@ func main() {
} }
if *runv5 { if *runv5 {
if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil { if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
utils.Fatalf("%v", err) utils.Fatalf("%v", err)
} }
} else { } else {
db, _ := enode.OpenDB("")
ln := enode.NewLocalNode(db, nodeKey)
cfg := discover.Config{ cfg := discover.Config{
PrivateKey: nodeKey, PrivateKey: nodeKey,
AnnounceAddr: realaddr, NetRestrict: restrictList,
NetRestrict: restrictList,
} }
if _, err := discover.ListenUDP(conn, cfg); err != nil { if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
utils.Fatalf("%v", err) utils.Fatalf("%v", err)
} }
} }

View File

@ -91,7 +91,7 @@ invoking methods with the following info:
* [x] Version info about the signer * [x] Version info about the signer
* [x] Address of API (http/ipc) * [x] Address of API (http/ipc)
* [ ] List of known accounts * [ ] List of known accounts
* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected. * [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
* [ ] `account_signRawTransaction` * [ ] `account_signRawTransaction`
* [ ] `account_bulkSignTransactions([] transactions)` should * [ ] `account_bulkSignTransactions([] transactions)` should
* only exist if enabled via config/flag * only exist if enabled via config/flag
@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification). expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
Some of these call can require user interaction. Clients must be aware that responses Some of these call can require user interaction. Clients must be aware that responses
may be delayed significanlty or may never be received if a users decides to ignore the confirmation request. may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
The External API is **untrusted** : it does not accept credentials over this api, nor does it expect The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
that requests have any authority. that requests have any authority.
@ -862,7 +862,7 @@ A UI should conform to the following rules.
* A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
* A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
* The signer provides accounts * The signer provides accounts
* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled * A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
along with the UI. along with the UI.
@ -875,3 +875,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)| | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)|
| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: | | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: |
| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: | | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: |
| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)|

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 36 KiB

View File

@ -1,6 +1,13 @@
### Changelog for external API ### Changelog for external API
#### 4.0.0
* The external `account_Ecrecover`-method was removed.
* The external `account_Import`-method was removed.
#### 3.0.0
* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses.
#### 2.0.0 #### 2.0.0

View File

@ -1,5 +1,24 @@
### Changelog for internal API (ui-api) ### Changelog for internal API (ui-api)
### 3.0.0
* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
### 2.1.0
* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
The following structures are used:
```golang
UserInputRequest struct {
Prompt string `json:"prompt"`
Title string `json:"title"`
IsPassword bool `json:"isPassword"`
}
UserInputResponse struct {
Text string `json:"text"`
}
### 2.0.0 ### 2.0.0
* Modify how `call_info` on a transaction is conveyed. New format: * Modify how `call_info` on a transaction is conveyed. New format:

View File

@ -35,8 +35,10 @@ import (
"runtime" "runtime"
"strings" "strings"
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/console"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
@ -48,10 +50,10 @@ import (
) )
// ExternalAPIVersion -- see extapi_changelog.md // ExternalAPIVersion -- see extapi_changelog.md
const ExternalAPIVersion = "2.0.0" const ExternalAPIVersion = "4.0.0"
// InternalAPIVersion -- see intapi_changelog.md // InternalAPIVersion -- see intapi_changelog.md
const InternalAPIVersion = "2.0.0" const InternalAPIVersion = "3.0.0"
const legalWarning = ` const legalWarning = `
WARNING! WARNING!
@ -70,6 +72,10 @@ var (
Value: 4, Value: 4,
Usage: "log level to emit to the screen", Usage: "log level to emit to the screen",
} }
advancedMode = cli.BoolFlag{
Name: "advanced",
Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
}
keystoreFlag = cli.StringFlag{ keystoreFlag = cli.StringFlag{
Name: "keystore", Name: "keystore",
Value: filepath.Join(node.DefaultDataDir(), "keystore"), Value: filepath.Join(node.DefaultDataDir(), "keystore"),
@ -87,7 +93,7 @@ var (
} }
signerSecretFlag = cli.StringFlag{ signerSecretFlag = cli.StringFlag{
Name: "signersecret", Name: "signersecret",
Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash", Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
} }
dBFlag = cli.StringFlag{ dBFlag = cli.StringFlag{
Name: "4bytedb", Name: "4bytedb",
@ -151,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
Clef that the file is 'safe' to execute.`, Clef that the file is 'safe' to execute.`,
} }
addCredentialCommand = cli.Command{ setCredentialCommand = cli.Command{
Action: utils.MigrateFlags(addCredential), Action: utils.MigrateFlags(setCredential),
Name: "addpw", Name: "setpw",
Usage: "Store a credential for a keystore file", Usage: "Store a credential for a keystore file",
ArgsUsage: "<address> <password>", ArgsUsage: "<address>",
Flags: []cli.Flag{ Flags: []cli.Flag{
logLevelFlag, logLevelFlag,
configdirFlag, configdirFlag,
signerSecretFlag, signerSecretFlag,
}, },
Description: ` Description: `
The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will
remove any stored credential for that address (keyfile) remove any stored credential for that address (keyfile)
`, `,
} }
@ -191,9 +197,10 @@ func init() {
ruleFlag, ruleFlag,
stdiouiFlag, stdiouiFlag,
testFlag, testFlag,
advancedMode,
} }
app.Action = signer app.Action = signer
app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand} app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
} }
func main() { func main() {
@ -207,25 +214,45 @@ func initializeSecrets(c *cli.Context) error {
if err := initialize(c); err != nil { if err := initialize(c); err != nil {
return err return err
} }
configDir := c.String(configdirFlag.Name) configDir := c.GlobalString(configdirFlag.Name)
masterSeed := make([]byte, 256) masterSeed := make([]byte, 256)
n, err := io.ReadFull(rand.Reader, masterSeed) num, err := io.ReadFull(rand.Reader, masterSeed)
if err != nil { if err != nil {
return err return err
} }
if n != len(masterSeed) { if num != len(masterSeed) {
return fmt.Errorf("failed to read enough random") return fmt.Errorf("failed to read enough random")
} }
n, p := keystore.StandardScryptN, keystore.StandardScryptP
if c.GlobalBool(utils.LightKDFFlag.Name) {
n, p = keystore.LightScryptN, keystore.LightScryptP
}
text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
var password string
for {
password = getPassPhrase(text, true)
if err := core.ValidatePasswordFormat(password); err != nil {
fmt.Printf("invalid password: %v\n", err)
} else {
break
}
}
cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p)
if err != nil {
return fmt.Errorf("failed to encrypt master seed: %v", err)
}
err = os.Mkdir(configDir, 0700) err = os.Mkdir(configDir, 0700)
if err != nil && !os.IsExist(err) { if err != nil && !os.IsExist(err) {
return err return err
} }
location := filepath.Join(configDir, "secrets.dat") location := filepath.Join(configDir, "masterseed.json")
if _, err := os.Stat(location); err == nil { if _, err := os.Stat(location); err == nil {
return fmt.Errorf("file %v already exists, will not overwrite", location) return fmt.Errorf("file %v already exists, will not overwrite", location)
} }
err = ioutil.WriteFile(location, masterSeed, 0700) err = ioutil.WriteFile(location, cipherSeed, 0400)
if err != nil { if err != nil {
return err return err
} }
@ -250,11 +277,11 @@ func attestFile(ctx *cli.Context) error {
return err return err
} }
stretchedKey, err := readMasterKey(ctx) stretchedKey, err := readMasterKey(ctx, nil)
if err != nil { if err != nil {
utils.Fatalf(err.Error()) utils.Fatalf(err.Error())
} }
configDir := ctx.String(configdirFlag.Name) configDir := ctx.GlobalString(configdirFlag.Name)
vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
confKey := crypto.Keccak256([]byte("config"), stretchedKey) confKey := crypto.Keccak256([]byte("config"), stretchedKey)
@ -266,38 +293,36 @@ func attestFile(ctx *cli.Context) error {
return nil return nil
} }
func addCredential(ctx *cli.Context) error { func setCredential(ctx *cli.Context) error {
if len(ctx.Args()) < 1 { if len(ctx.Args()) < 1 {
utils.Fatalf("This command requires at leaste one argument.") utils.Fatalf("This command requires an address to be passed as an argument.")
} }
if err := initialize(ctx); err != nil { if err := initialize(ctx); err != nil {
return err return err
} }
stretchedKey, err := readMasterKey(ctx) address := ctx.Args().First()
password := getPassPhrase("Enter a passphrase to store with this address.", true)
stretchedKey, err := readMasterKey(ctx, nil)
if err != nil { if err != nil {
utils.Fatalf(err.Error()) utils.Fatalf(err.Error())
} }
configDir := ctx.String(configdirFlag.Name) configDir := ctx.GlobalString(configdirFlag.Name)
vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
// Initialize the encrypted storages // Initialize the encrypted storages
pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey) pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
key := ctx.Args().First() pwStorage.Put(address, password)
value := "" log.Info("Credential store updated", "key", address)
if len(ctx.Args()) > 1 {
value = ctx.Args().Get(1)
}
pwStorage.Put(key, value)
log.Info("Credential store updated", "key", key)
return nil return nil
} }
func initialize(c *cli.Context) error { func initialize(c *cli.Context) error {
// Set up the logger to print everything // Set up the logger to print everything
logOutput := os.Stdout logOutput := os.Stdout
if c.Bool(stdiouiFlag.Name) { if c.GlobalBool(stdiouiFlag.Name) {
logOutput = os.Stderr logOutput = os.Stderr
// If using the stdioui, we can't do the 'confirm'-flow // If using the stdioui, we can't do the 'confirm'-flow
fmt.Fprintf(logOutput, legalWarning) fmt.Fprintf(logOutput, legalWarning)
@ -318,26 +343,28 @@ func signer(c *cli.Context) error {
var ( var (
ui core.SignerUI ui core.SignerUI
) )
if c.Bool(stdiouiFlag.Name) { if c.GlobalBool(stdiouiFlag.Name) {
log.Info("Using stdin/stdout as UI-channel") log.Info("Using stdin/stdout as UI-channel")
ui = core.NewStdIOUI() ui = core.NewStdIOUI()
} else { } else {
log.Info("Using CLI as UI-channel") log.Info("Using CLI as UI-channel")
ui = core.NewCommandlineUI() ui = core.NewCommandlineUI()
} }
db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name)) fourByteDb := c.GlobalString(dBFlag.Name)
fourByteLocal := c.GlobalString(customDBFlag.Name)
db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal)
if err != nil { if err != nil {
utils.Fatalf(err.Error()) utils.Fatalf(err.Error())
} }
log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb")) log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal)
var ( var (
api core.ExternalAPI api core.ExternalAPI
) )
configDir := c.String(configdirFlag.Name) configDir := c.GlobalString(configdirFlag.Name)
if stretchedKey, err := readMasterKey(c); err != nil { if stretchedKey, err := readMasterKey(c, ui); err != nil {
log.Info("No master seed provided, rules disabled") log.Info("No master seed provided, rules disabled", "error", err)
} else { } else {
if err != nil { if err != nil {
@ -356,7 +383,7 @@ func signer(c *cli.Context) error {
configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
//Do we have a rule-file? //Do we have a rule-file?
ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name)) ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name))
if err != nil { if err != nil {
log.Info("Could not load rulefile, rules not enabled", "file", "rulefile") log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
} else { } else {
@ -380,16 +407,15 @@ func signer(c *cli.Context) error {
} }
apiImpl := core.NewSignerAPI( apiImpl := core.NewSignerAPI(
c.Int64(utils.NetworkIdFlag.Name), c.GlobalInt64(utils.NetworkIdFlag.Name),
c.String(keystoreFlag.Name), c.GlobalString(keystoreFlag.Name),
c.Bool(utils.NoUSBFlag.Name), c.GlobalBool(utils.NoUSBFlag.Name),
ui, db, ui, db,
c.Bool(utils.LightKDFFlag.Name)) c.GlobalBool(utils.LightKDFFlag.Name),
c.GlobalBool(advancedMode.Name))
api = apiImpl api = apiImpl
// Audit logging // Audit logging
if logfile := c.String(auditLogFlag.Name); logfile != "" { if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
api, err = core.NewAuditLogger(logfile, api) api, err = core.NewAuditLogger(logfile, api)
if err != nil { if err != nil {
utils.Fatalf(err.Error()) utils.Fatalf(err.Error())
@ -408,13 +434,13 @@ func signer(c *cli.Context) error {
Service: api, Service: api,
Version: "1.0"}, Version: "1.0"},
} }
if c.Bool(utils.RPCEnabledFlag.Name) { if c.GlobalBool(utils.RPCEnabledFlag.Name) {
vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name)) vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name)) cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
// start http server // start http server
httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name)) httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts) listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
if err != nil { if err != nil {
utils.Fatalf("Could not start RPC api: %v", err) utils.Fatalf("Could not start RPC api: %v", err)
@ -428,9 +454,9 @@ func signer(c *cli.Context) error {
}() }()
} }
if !c.Bool(utils.IPCDisabledFlag.Name) { if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
if c.IsSet(utils.IPCPathFlag.Name) { if c.IsSet(utils.IPCPathFlag.Name) {
ipcapiURL = c.String(utils.IPCPathFlag.Name) ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name)
} else { } else {
ipcapiURL = filepath.Join(configDir, "clef.ipc") ipcapiURL = filepath.Join(configDir, "clef.ipc")
} }
@ -447,7 +473,7 @@ func signer(c *cli.Context) error {
} }
if c.Bool(testFlag.Name) { if c.GlobalBool(testFlag.Name) {
log.Info("Performing UI test") log.Info("Performing UI test")
go testExternalUI(apiImpl) go testExternalUI(apiImpl)
} }
@ -506,48 +532,64 @@ func homeDir() string {
} }
return "" return ""
} }
func readMasterKey(ctx *cli.Context) ([]byte, error) { func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
var ( var (
file string file string
configDir = ctx.String(configdirFlag.Name) configDir = ctx.GlobalString(configdirFlag.Name)
) )
if ctx.IsSet(signerSecretFlag.Name) { if ctx.GlobalIsSet(signerSecretFlag.Name) {
file = ctx.String(signerSecretFlag.Name) file = ctx.GlobalString(signerSecretFlag.Name)
} else { } else {
file = filepath.Join(configDir, "secrets.dat") file = filepath.Join(configDir, "masterseed.json")
} }
if err := checkFile(file); err != nil { if err := checkFile(file); err != nil {
return nil, err return nil, err
} }
masterKey, err := ioutil.ReadFile(file) cipherKey, err := ioutil.ReadFile(file)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(masterKey) < 256 { var password string
return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey)) // If ui is not nil, get the password from ui.
if ui != nil {
resp, err := ui.OnInputRequired(core.UserInputRequest{
Title: "Master Password",
Prompt: "Please enter the password to decrypt the master seed",
IsPassword: true})
if err != nil {
return nil, err
}
password = resp.Text
} else {
password = getPassPhrase("Decrypt master seed of clef", false)
} }
masterSeed, err := decryptSeed(cipherKey, password)
if err != nil {
return nil, fmt.Errorf("failed to decrypt the master seed of clef")
}
if len(masterSeed) < 256 {
return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
}
// Create vault location // Create vault location
vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10])) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
err = os.Mkdir(vaultLocation, 0700) err = os.Mkdir(vaultLocation, 0700)
if err != nil && !os.IsExist(err) { if err != nil && !os.IsExist(err) {
return nil, err return nil, err
} }
//!TODO, use KDF to stretch the master key return masterSeed, nil
// stretched_key := stretch_key(master_key)
return masterKey, nil
} }
// checkFile is a convenience function to check if a file // checkFile is a convenience function to check if a file
// * exists // * exists
// * is mode 0600 // * is mode 0400
func checkFile(filename string) error { func checkFile(filename string) error {
info, err := os.Stat(filename) info, err := os.Stat(filename)
if err != nil { if err != nil {
return fmt.Errorf("failed stat on %s: %v", filename, err) return fmt.Errorf("failed stat on %s: %v", filename, err)
} }
// Check the unix permission bits // Check the unix permission bits
if info.Mode().Perm()&077 != 0 { if info.Mode().Perm()&0377 != 0 {
return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String()) return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
} }
return nil return nil
@ -613,6 +655,59 @@ func testExternalUI(api *core.SignerAPI) {
} }
// getPassPhrase retrieves the password associated with clef, either fetched
// from a list of preloaded passphrases, or requested interactively from the user.
// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
func getPassPhrase(prompt string, confirmation bool) string {
fmt.Println(prompt)
password, err := console.Stdin.PromptPassword("Passphrase: ")
if err != nil {
utils.Fatalf("Failed to read passphrase: %v", err)
}
if confirmation {
confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
if err != nil {
utils.Fatalf("Failed to read passphrase confirmation: %v", err)
}
if password != confirm {
utils.Fatalf("Passphrases do not match")
}
}
return password
}
type encryptedSeedStorage struct {
Description string `json:"description"`
Version int `json:"version"`
Params keystore.CryptoJSON `json:"params"`
}
// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping,
// to encrypt the master seed
func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) {
cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP)
if err != nil {
return nil, err
}
return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct})
}
// decryptSeed decrypts the master seed
func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
var encSeed encryptedSeedStorage
if err := json.Unmarshal(keyjson, &encSeed); err != nil {
return nil, err
}
if encSeed.Version != 1 {
log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version))
}
seed, err := keystore.DecryptDataV3(encSeed.Params, auth)
if err != nil {
return nil, err
}
return seed, err
}
/** /**
//Create Account //Create Account

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 20 KiB

View File

@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ
## Creating rules ## Creating rules
Now, you can create a rule-file. Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
```javascript ```javascript
function ApproveListing(){ function ApproveListing(){
return "Approve" return "Approve"
} }
``` ```
Get the `sha256` hash....
Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
```text ```text
#sha256sum rules.js #sha256sum rules.js
6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 rules.js 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 rules.js
``` ```
...And then `attest` the file: ...now `attest` the file...
```text ```text
#./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
INFO [02-21|12:14:38] Ruleset attestation updated sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72 INFO [02-21|12:14:38] Ruleset attestation updated sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
``` ```
At this point, we then start the signer with the rule-file:
...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
```text ```text
#./signer --rules rules.json #cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
```
INFO [02-21|12:15:18] Using CLI as UI-channel At this point, we can start the signer with the rule-file:
INFO [02-21|12:15:18] Loaded 4byte db signatures=5509 file=./4byte.json ```text
INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile #./signer --rules rules.js --rpc
DEBUG[02-21|12:15:18] FS scan times list=35.335µs set=5.536µs diff=5.073µs
DEBUG[02-21|12:15:18] Ledger support enabled INFO [09-25|20:28:11.866] Using CLI as UI-channel
DEBUG[02-21|12:15:18] Trezor support enabled INFO [09-25|20:28:11.876] Loaded 4byte db signatures=5509 file=./4byte.json
INFO [02-21|12:15:18] Audit logs configured file=audit.log INFO [09-25|20:28:11.877] Rule engine configured file=./rules.js
INFO [02-21|12:15:18] HTTP endpoint opened url=http://localhost:8550 DEBUG[09-25|20:28:11.877] FS scan times list=100.781µs set=13.253µs diff=5.761µs
DEBUG[09-25|20:28:11.884] Ledger support enabled
DEBUG[09-25|20:28:11.888] Trezor support enabled
INFO [09-25|20:28:11.888] Audit logs configured file=audit.log
DEBUG[09-25|20:28:11.888] HTTP registered namespace=account
INFO [09-25|20:28:11.890] HTTP endpoint opened url=http://localhost:8550
DEBUG[09-25|20:28:11.890] IPC registered namespace=account
INFO [09-25|20:28:11.890] IPC endpoint opened url=<nil>
------- Signer info ------- ------- Signer info -------
* extapi_version : 2.0.0
* intapi_version : 2.0.0
* extapi_http : http://localhost:8550 * extapi_http : http://localhost:8550
* extapi_ipc : <nil> * extapi_ipc : <nil>
* extapi_version : 2.0.0
* intapi_version : 1.2.0
``` ```
Any list-requests will now be auto-approved by our rule-file. Any list-requests will now be auto-approved by our rule-file.
@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things:
## Adding credentials ## Adding credentials
In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores. In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
```text ```text
#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test #./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
INFO [02-21|13:43:21] Credential store updated key=0x694267f14675d7e1b9494fd8d72fefe1755710fa INFO [02-21|13:43:21] Credential store updated key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
``` ```
## More advanced rules ## More advanced rules
Now let's update the rules to make use of credentials Now let's update the rules to make use of credentials:
```javascript ```javascript
function ApproveListing(){ function ApproveListing(){
@ -134,13 +142,15 @@ function ApproveSignData(r){
} }
``` ```
In this example, In this example:
* any requests to sign data with the account `0x694...` will be * Any requests to sign data with the account `0x694...` will be
* auto-approved if the message contains with `bazonk`, * auto-approved if the message contains with `bazonk`
* and auto-rejected if it does not. * auto-rejected if it does not.
* Any other signing-requests will be passed along for manual approve/reject. * Any other signing-requests will be passed along for manual approve/reject.
..attest the new file _Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
Attest the new file...
```text ```text
#sha256sum rules.js #sha256sum rules.js
2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f rules.js 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f rules.js
@ -153,23 +163,26 @@ INFO [02-21|14:36:30] Ruleset attestation updated sha256=2a0cb661da
And start the signer: And start the signer:
``` ```
#./signer --rules rules.js #./signer --rules rules.js --rpc
INFO [02-21|14:41:56] Using CLI as UI-channel INFO [09-25|21:02:16.450] Using CLI as UI-channel
INFO [02-21|14:41:56] Loaded 4byte db signatures=5509 file=./4byte.json INFO [09-25|21:02:16.466] Loaded 4byte db signatures=5509 file=./4byte.json
INFO [02-21|14:41:56] Rule engine configured file=rules.js INFO [09-25|21:02:16.467] Rule engine configured file=./rules.js
DEBUG[02-21|14:41:56] FS scan times list=34.607µs set=4.509µs diff=4.87µs DEBUG[09-25|21:02:16.468] FS scan times list=1.45262ms set=21.926µs diff=6.944µs
DEBUG[02-21|14:41:56] Ledger support enabled DEBUG[09-25|21:02:16.473] Ledger support enabled
DEBUG[02-21|14:41:56] Trezor support enabled DEBUG[09-25|21:02:16.475] Trezor support enabled
INFO [02-21|14:41:56] Audit logs configured file=audit.log INFO [09-25|21:02:16.476] Audit logs configured file=audit.log
INFO [02-21|14:41:56] HTTP endpoint opened url=http://localhost:8550 DEBUG[09-25|21:02:16.476] HTTP registered namespace=account
INFO [09-25|21:02:16.478] HTTP endpoint opened url=http://localhost:8550
DEBUG[09-25|21:02:16.478] IPC registered namespace=account
INFO [09-25|21:02:16.478] IPC endpoint opened url=<nil>
------- Signer info ------- ------- Signer info -------
* extapi_version : 2.0.0 * extapi_version : 2.0.0
* intapi_version : 1.2.0 * intapi_version : 2.0.0
* extapi_http : http://localhost:8550 * extapi_http : http://localhost:8550
* extapi_ipc : <nil> * extapi_ipc : <nil>
INFO [02-21|14:41:56] error occurred during execution error="ReferenceError: 'OnSignerStartup' is not defined"
``` ```
And then test signing, once with `bazonk` and once without: And then test signing, once with `bazonk` and once without:
``` ```
@ -190,9 +203,9 @@ INFO [02-21|14:42:56] Op rejected
The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses: The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
```text ```text
#tail audit.log -n 4 #tail -n 4 audit.log
t=2018-02-21T14:42:41+0100 lvl=info msg=Sign api=signer type=request metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a t=2018-02-21T14:42:41+0100 lvl=info msg=Sign api=signer type=request metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
t=2018-02-21T14:42:42+0100 lvl=info msg=Sign api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil t=2018-02-21T14:42:42+0100 lvl=info msg=Sign api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
t=2018-02-21T14:42:56+0100 lvl=info msg=Sign api=signer type=request metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a t=2018-02-21T14:42:56+0100 lvl=info msg=Sign api=signer type=request metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
t=2018-02-21T14:42:56+0100 lvl=info msg=Sign api=signer type=response data= error="Request denied" t=2018-02-21T14:42:56+0100 lvl=info msg=Sign api=signer type=response data= error="Request denied"
``` ```

View File

@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag;
make sure to use this feature with great caution! make sure to use this feature with great caution!
### `ethkey sign <keyfile> <message/file>` ### `ethkey signmessage <keyfile> <message/file>`
Sign the message with a keyfile. Sign the message with a keyfile.
It is possible to refer to a file containing the message. It is possible to refer to a file containing the message.
To sign a message contained in a file, use the `--msgfile` flag.
### `ethkey verify <address> <signature> <message/file>` ### `ethkey verifymessage <address> <signature> <message/file>`
Verify the signature of the message. Verify the signature of the message.
It is possible to refer to a file containing the message. It is possible to refer to a file containing the message.
To sign a message contained in a file, use the --msgfile flag.
### `ethkey changepassphrase <keyfile>`
Change the passphrase of a keyfile.
use the `--newpasswordfile` to point to the new password file.
## Passphrases ## Passphrases
For every command that uses a keyfile, you will be prompted to provide the For every command that uses a keyfile, you will be prompted to provide the
passphrase for decrypting the keyfile. To avoid this message, it is possible passphrase for decrypting the keyfile. To avoid this message, it is possible
to pass the passphrase by using the `--passphrase` flag pointing to a file that to pass the passphrase by using the `--passwordfile` flag pointing to a file that
contains the passphrase. contains the passphrase.
## JSON
In case you need to output the result in a JSON format, you shall by using the `--json` flag.

View File

@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error {
return err return err
} }
code := strings.TrimSpace(string(in[:])) code := strings.TrimSpace(string(in))
fmt.Printf("%v\n", code) fmt.Printf("%v\n", code)
return asm.PrintDisassembled(code) return asm.PrintDisassembled(code)
} }

View File

@ -45,14 +45,15 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create
// CaptureState outputs state information on the logger. // CaptureState outputs state information on the logger.
func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error {
log := vm.StructLog{ log := vm.StructLog{
Pc: pc, Pc: pc,
Op: op, Op: op,
Gas: gas, Gas: gas,
GasCost: cost, GasCost: cost,
MemorySize: memory.Len(), MemorySize: memory.Len(),
Storage: nil, Storage: nil,
Depth: depth, Depth: depth,
Err: err, RefundCounter: env.StateDB.GetRefund(),
Err: err,
} }
if !l.cfg.DisableMemory { if !l.cfg.DisableMemory {
log.Memory = memory.Data() log.Memory = memory.Data()

View File

@ -80,13 +80,13 @@ func runCmd(ctx *cli.Context) error {
} }
var ( var (
tracer vm.Tracer tracer vm.Tracer
debugLogger *vm.StructLogger debugLogger *vm.StructLogger
statedb *state.StateDB statedb *state.StateDB
chainConfig *params.ChainConfig chainConfig *params.ChainConfig
sender = common.BytesToAddress([]byte("sender")) sender = common.BytesToAddress([]byte("sender"))
receiver = common.BytesToAddress([]byte("receiver")) receiver = common.BytesToAddress([]byte("receiver"))
blockNumber uint64 genesisConfig *core.Genesis
) )
if ctx.GlobalBool(MachineFlag.Name) { if ctx.GlobalBool(MachineFlag.Name) {
tracer = NewJSONLogger(logconfig, os.Stdout) tracer = NewJSONLogger(logconfig, os.Stdout)
@ -98,13 +98,14 @@ func runCmd(ctx *cli.Context) error {
} }
if ctx.GlobalString(GenesisFlag.Name) != "" { if ctx.GlobalString(GenesisFlag.Name) != "" {
gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
genesisConfig = gen
db := ethdb.NewMemDatabase() db := ethdb.NewMemDatabase()
genesis := gen.ToBlock(db) genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db)) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
chainConfig = gen.Config chainConfig = gen.Config
blockNumber = gen.Number
} else { } else {
statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase())) statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
genesisConfig = new(core.Genesis)
} }
if ctx.GlobalString(SenderFlag.Name) != "" { if ctx.GlobalString(SenderFlag.Name) != "" {
sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
@ -156,13 +157,19 @@ func runCmd(ctx *cli.Context) error {
} }
initialGas := ctx.GlobalUint64(GasFlag.Name) initialGas := ctx.GlobalUint64(GasFlag.Name)
if genesisConfig.GasLimit != 0 {
initialGas = genesisConfig.GasLimit
}
runtimeConfig := runtime.Config{ runtimeConfig := runtime.Config{
Origin: sender, Origin: sender,
State: statedb, State: statedb,
GasLimit: initialGas, GasLimit: initialGas,
GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
Value: utils.GlobalBig(ctx, ValueFlag.Name), Value: utils.GlobalBig(ctx, ValueFlag.Name),
BlockNumber: new(big.Int).SetUint64(blockNumber), Difficulty: genesisConfig.Difficulty,
Time: new(big.Int).SetUint64(genesisConfig.Timestamp),
Coinbase: genesisConfig.Coinbase,
BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
EVMConfig: vm.Config{ EVMConfig: vm.Config{
Tracer: tracer, Tracer: tracer,
Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),

View File

@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error {
// Run the test and aggregate the result // Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
state, err := test.Run(st, cfg) state, err := test.Run(st, cfg)
// print state root for evmlab tracing
if ctx.GlobalBool(MachineFlag.Name) && state != nil {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
}
if err != nil { if err != nil {
// Test failed, mark as so and dump any state to aid debugging // Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error() result.Pass, result.Error = false, err.Error()
@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error {
result.State = &dump result.State = &dump
} }
} }
// print state root for evmlab tracing (already committed above, so no need to delete objects again
if ctx.GlobalBool(MachineFlag.Name) && state != nil {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
}
results = append(results, *result) results = append(results, *result)

View File

@ -54,8 +54,8 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/p2p/discv5"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/nat"
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
"golang.org/x/net/websocket" "golang.org/x/net/websocket"
@ -157,7 +157,8 @@ func main() {
if blob, err = ioutil.ReadFile(*accPassFlag); err != nil { if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err) log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
} }
pass := string(blob) // Delete trailing newline in password
pass := strings.TrimSuffix(string(blob), "\n")
ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP) ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil { if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
@ -198,6 +199,8 @@ type faucet struct {
keystore *keystore.KeyStore // Keystore containing the single signer keystore *keystore.KeyStore // Keystore containing the single signer
account accounts.Account // Account funding user faucet requests account accounts.Account // Account funding user faucet requests
head *types.Header // Current head header of the faucet
balance *big.Int // Current balance of the faucet
nonce uint64 // Current pending nonce of the faucet nonce uint64 // Current pending nonce of the faucet
price *big.Int // Current gas price to issue funds with price *big.Int // Current gas price to issue funds with
@ -252,8 +255,10 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
return nil, err return nil, err
} }
for _, boot := range enodes { for _, boot := range enodes {
old, _ := discover.ParseNode(boot.String()) old, err := enode.ParseV4(boot.String())
stack.Server().AddPeer(old) if err != nil {
stack.Server().AddPeer(old)
}
} }
// Attach to the client and retrieve and interesting metadatas // Attach to the client and retrieve and interesting metadatas
api, err := stack.Attach() api, err := stack.Attach()
@ -323,33 +328,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
nonce uint64 nonce uint64
err error err error
) )
for { for head == nil || balance == nil {
// Attempt to retrieve the stats, may error on no faucet connectivity // Retrieve the current stats cached by the faucet
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) f.lock.RLock()
head, err = f.client.HeaderByNumber(ctx, nil) if f.head != nil {
if err == nil { head = types.CopyHeader(f.head)
balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
if err == nil {
nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
}
} }
cancel() if f.balance != nil {
balance = new(big.Int).Set(f.balance)
}
nonce = f.nonce
f.lock.RUnlock()
// If stats retrieval failed, wait a bit and retry if head == nil || balance == nil {
if err != nil { // Report the faucet offline until initial stats are ready
if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil { if err = sendError(conn, errors.New("Faucet offline")); err != nil {
log.Warn("Failed to send faucet error to client", "err", err) log.Warn("Failed to send faucet error to client", "err", err)
return return
} }
time.Sleep(3 * time.Second) time.Sleep(3 * time.Second)
continue
} }
// Initial stats reported successfully, proceed with user interaction
break
} }
// Send over the initial stats and the latest header // Send over the initial stats and the latest header
if err = send(conn, map[string]interface{}{ if err = send(conn, map[string]interface{}{
"funds": balance.Div(balance, ether), "funds": new(big.Int).Div(balance, ether),
"funded": nonce, "funded": nonce,
"peers": f.stack.Server().PeerCount(), "peers": f.stack.Server().PeerCount(),
"requests": f.reqs, "requests": f.reqs,
@ -519,6 +521,47 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
} }
} }
// refresh attempts to retrieve the latest header from the chain and extract the
// associated faucet balance and nonce for connectivity caching.
func (f *faucet) refresh(head *types.Header) error {
// Ensure a state update does not run for too long
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// If no header was specified, use the current chain head
var err error
if head == nil {
if head, err = f.client.HeaderByNumber(ctx, nil); err != nil {
return err
}
}
// Retrieve the balance, nonce and gas price from the current head
var (
balance *big.Int
nonce uint64
price *big.Int
)
if balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number); err != nil {
return err
}
if nonce, err = f.client.NonceAt(ctx, f.account.Address, head.Number); err != nil {
return err
}
if price, err = f.client.SuggestGasPrice(ctx); err != nil {
return err
}
// Everything succeeded, update the cached stats and eject old requests
f.lock.Lock()
f.head, f.balance = head, balance
f.price, f.nonce = price, nonce
for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
f.reqs = f.reqs[1:]
}
f.lock.Unlock()
return nil
}
// loop keeps waiting for interesting events and pushes them out to connected // loop keeps waiting for interesting events and pushes them out to connected
// websockets. // websockets.
func (f *faucet) loop() { func (f *faucet) loop() {
@ -536,45 +579,27 @@ func (f *faucet) loop() {
go func() { go func() {
for head := range update { for head := range update {
// New chain head arrived, query the current stats and stream to clients // New chain head arrived, query the current stats and stream to clients
var ( timestamp := time.Unix(head.Time.Int64(), 0)
balance *big.Int if time.Since(timestamp) > time.Hour {
nonce uint64 log.Warn("Skipping faucet refresh, head too old", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp))
price *big.Int continue
err error
)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
if err == nil {
nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
if err == nil {
price, err = f.client.SuggestGasPrice(ctx)
}
} }
cancel() if err := f.refresh(head); err != nil {
// If querying the data failed, try for the next block
if err != nil {
log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err) log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
continue continue
} else {
log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
} }
// Faucet state retrieved, update locally and send to clients // Faucet state retrieved, update locally and send to clients
balance = new(big.Int).Div(balance, ether)
f.lock.Lock()
f.price, f.nonce = price, nonce
for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
f.reqs = f.reqs[1:]
}
f.lock.Unlock()
f.lock.RLock() f.lock.RLock()
log.Info("Updated faucet state", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp), "balance", f.balance, "nonce", f.nonce, "price", f.price)
balance := new(big.Int).Div(f.balance, ether)
peers := f.stack.Server().PeerCount()
for _, conn := range f.conns { for _, conn := range f.conns {
if err := send(conn, map[string]interface{}{ if err := send(conn, map[string]interface{}{
"funds": balance, "funds": balance,
"funded": f.nonce, "funded": f.nonce,
"peers": f.stack.Server().PeerCount(), "peers": peers,
"requests": f.reqs, "requests": f.reqs,
}, time.Second); err != nil { }, time.Second); err != nil {
log.Warn("Failed to send stats to client", "err", err) log.Warn("Failed to send stats to client", "err", err)

View File

@ -340,9 +340,9 @@ func importPreimages(ctx *cli.Context) error {
start := time.Now() start := time.Now()
if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil { if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
utils.Fatalf("Export error: %v\n", err) utils.Fatalf("Import error: %v\n", err)
} }
fmt.Printf("Export done in %v\n", time.Since(start)) fmt.Printf("Import done in %v\n", time.Since(start))
return nil return nil
} }

View File

@ -168,6 +168,9 @@ func makeFullNode(ctx *cli.Context) *node.Node {
if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) { if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) {
cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name) cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name)
} }
if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
cfg.Shh.RestrictConnectionBetweenLightClients = true
}
utils.RegisterShhService(stack, &cfg.Shh) utils.RegisterShhService(stack, &cfg.Shh)
} }

View File

@ -21,7 +21,6 @@ import (
"fmt" "fmt"
"math" "math"
"os" "os"
"runtime"
godebug "runtime/debug" godebug "runtime/debug"
"sort" "sort"
"strconv" "strconv"
@ -130,6 +129,8 @@ var (
utils.NoCompactionFlag, utils.NoCompactionFlag,
utils.GpoBlocksFlag, utils.GpoBlocksFlag,
utils.GpoPercentileFlag, utils.GpoPercentileFlag,
utils.EWASMInterpreterFlag,
utils.EVMInterpreterFlag,
configFileFlag, configFileFlag,
} }
@ -151,6 +152,7 @@ var (
utils.WhisperEnabledFlag, utils.WhisperEnabledFlag,
utils.WhisperMaxMessageSizeFlag, utils.WhisperMaxMessageSizeFlag,
utils.WhisperMinPOWFlag, utils.WhisperMinPOWFlag,
utils.WhisperRestrictConnectionBetweenLightClientsFlag,
} }
metricsFlags = []cli.Flag{ metricsFlags = []cli.Flag{
@ -206,8 +208,6 @@ func init() {
app.Flags = append(app.Flags, metricsFlags...) app.Flags = append(app.Flags, metricsFlags...)
app.Before = func(ctx *cli.Context) error { app.Before = func(ctx *cli.Context) error {
runtime.GOMAXPROCS(runtime.NumCPU())
logdir := "" logdir := ""
if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) { if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs") logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")

View File

@ -207,6 +207,8 @@ var AppHelpFlagGroups = []flagGroup{
Name: "VIRTUAL MACHINE", Name: "VIRTUAL MACHINE",
Flags: []cli.Flag{ Flags: []cli.Flag{
utils.VMEnableDebugFlag, utils.VMEnableDebugFlag,
utils.EVMInterpreterFlag,
utils.EWASMInterpreterFlag,
}, },
}, },
{ {

View File

@ -47,7 +47,7 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
@ -285,7 +285,7 @@ func createNode(ctx *cli.Context) error {
if err != nil { if err != nil {
return err return err
} }
config.ID = discover.PubkeyID(&privKey.PublicKey) config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
config.PrivateKey = privKey config.PrivateKey = privKey
} }
if services := ctx.String("services"); services != "" { if services := ctx.String("services"); services != "" {

View File

@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
RUN \ RUN \
echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
ENTRYPOINT ["/bin/sh", "geth.sh"] ENTRYPOINT ["/bin/sh", "geth.sh"]
` `
@ -99,6 +99,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{ template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
"NetworkID": config.network, "NetworkID": config.network,
"Port": config.port, "Port": config.port,
"IP": client.address,
"Peers": config.peersTotal, "Peers": config.peersTotal,
"LightFlag": lightFlag, "LightFlag": lightFlag,
"Bootnodes": strings.Join(bootnodes, ","), "Bootnodes": strings.Join(bootnodes, ","),
@ -227,10 +228,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
// Container available, retrieve its node ID and its genesis json // Container available, retrieve its node ID and its genesis json
var out []byte var out []byte
if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil { if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil {
return nil, ErrServiceUnreachable return nil, ErrServiceUnreachable
} }
id := bytes.Trim(bytes.TrimSpace(out), "\"") enode := bytes.Trim(bytes.TrimSpace(out), "\"")
if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil { if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
return nil, ErrServiceUnreachable return nil, ErrServiceUnreachable
@ -265,7 +266,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
gasLimit: gasLimit, gasLimit: gasLimit,
gasPrice: gasPrice, gasPrice: gasPrice,
} }
stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port) stats.enode = string(enode)
return stats, nil return stats, nil
} }

View File

@ -92,7 +92,7 @@ func (w *wizard) deployDashboard() {
pages = append(pages, page) pages = append(pages, page)
} }
} }
// Promt the user to chose one, enter manually or simply not list this service // Prompt the user to chose one, enter manually or simply not list this service
defLabel, defChoice := "don't list", len(pages)+2 defLabel, defChoice := "don't list", len(pages)+2
if len(pages) > 0 { if len(pages) > 0 {
defLabel, defChoice = pages[0], 1 defLabel, defChoice = pages[0], 1

View File

@ -87,7 +87,7 @@ func (w *wizard) makeServer() string {
return input return input
} }
// selectServer lists the user all the currnetly known servers to choose from, // selectServer lists the user all the currently known servers to choose from,
// also granting the option to add a new one. // also granting the option to add a new one.
func (w *wizard) selectServer() string { func (w *wizard) selectServer() string {
// List the available server to the user and wait for a choice // List the available server to the user and wait for a choice
@ -115,7 +115,7 @@ func (w *wizard) selectServer() string {
// manageComponents displays a list of network components the user can tear down // manageComponents displays a list of network components the user can tear down
// and an option // and an option
func (w *wizard) manageComponents() { func (w *wizard) manageComponents() {
// List all the componens we can tear down, along with an entry to deploy a new one // List all the components we can tear down, along with an entry to deploy a new one
fmt.Println() fmt.Println()
var serviceHosts, serviceNames []string var serviceHosts, serviceNames []string

View File

@ -29,7 +29,65 @@ import (
"gopkg.in/urfave/cli.v1" "gopkg.in/urfave/cli.v1"
) )
var salt = make([]byte, 32) var (
salt = make([]byte, 32)
accessCommand = cli.Command{
CustomHelpTemplate: helpTemplate,
Name: "access",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root manifest",
Subcommands: []cli.Command{
{
CustomHelpTemplate: helpTemplate,
Name: "new",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
Subcommands: []cli.Command{
{
Action: accessNewPass,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
},
Name: "pass",
Usage: "encrypts a reference with a password and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewPK,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
SwarmAccessGrantKeyFlag,
},
Name: "pk",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewACT,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
SwarmAccessGrantKeysFlag,
SwarmDryRunFlag,
utils.PasswordFileFlag,
},
Name: "act",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
},
},
},
}
)
func init() { func init() {
if _, err := io.ReadFull(rand.Reader, salt); err != nil { if _, err := io.ReadFull(rand.Reader, salt); err != nil {
@ -51,18 +109,20 @@ func accessNewPass(ctx *cli.Context) {
password = getPassPhrase("", 0, makePasswordList(ctx)) password = getPassPhrase("", 0, makePasswordList(ctx))
dryRun = ctx.Bool(SwarmDryRunFlag.Name) dryRun = ctx.Bool(SwarmDryRunFlag.Name)
) )
accessKey, ae, err = api.DoPasswordNew(ctx, password, salt) accessKey, ae, err = api.DoPassword(ctx, password, salt)
if err != nil { if err != nil {
utils.Fatalf("error getting session key: %v", err) utils.Fatalf("error getting session key: %v", err)
} }
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if err != nil {
utils.Fatalf("had an error generating the manifest: %v", err)
}
if dryRun { if dryRun {
err = printManifests(m, nil) err = printManifests(m, nil)
if err != nil { if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err) utils.Fatalf("had an error printing the manifests: %v", err)
} }
} else { } else {
utils.Fatalf("uploading manifests")
err = uploadManifests(ctx, m, nil) err = uploadManifests(ctx, m, nil)
if err != nil { if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err) utils.Fatalf("had an error uploading the manifests: %v", err)
@ -85,11 +145,14 @@ func accessNewPK(ctx *cli.Context) {
granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name) granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
dryRun = ctx.Bool(SwarmDryRunFlag.Name) dryRun = ctx.Bool(SwarmDryRunFlag.Name)
) )
sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt) sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt)
if err != nil { if err != nil {
utils.Fatalf("error getting session key: %v", err) utils.Fatalf("error getting session key: %v", err)
} }
m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae) m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
if err != nil {
utils.Fatalf("had an error generating the manifest: %v", err)
}
if dryRun { if dryRun {
err = printManifests(m, nil) err = printManifests(m, nil)
if err != nil { if err != nil {
@ -110,23 +173,38 @@ func accessNewACT(ctx *cli.Context) {
} }
var ( var (
ae *api.AccessEntry ae *api.AccessEntry
actManifest *api.Manifest actManifest *api.Manifest
accessKey []byte accessKey []byte
err error err error
ref = args[0] ref = args[0]
grantees = []string{} pkGrantees = []string{}
actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) passGrantees = []string{}
privateKey = getPrivKey(ctx) pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
dryRun = ctx.Bool(SwarmDryRunFlag.Name) passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
privateKey = getPrivKey(ctx)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
) )
if pkGranteesFilename == "" && passGranteesFilename == "" {
bytes, err := ioutil.ReadFile(actFilename) utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)")
if err != nil {
utils.Fatalf("had an error reading the grantee public key list")
} }
grantees = strings.Split(string(bytes), "\n")
accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees) if pkGranteesFilename != "" {
bytes, err := ioutil.ReadFile(pkGranteesFilename)
if err != nil {
utils.Fatalf("had an error reading the grantee public key list")
}
pkGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
}
if passGranteesFilename != "" {
bytes, err := ioutil.ReadFile(passGranteesFilename)
if err != nil {
utils.Fatalf("could not read password filename: %v", err)
}
passGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
}
accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees)
if err != nil { if err != nil {
utils.Fatalf("error generating ACT manifest: %v", err) utils.Fatalf("error generating ACT manifest: %v", err)
} }

View File

@ -28,50 +28,43 @@ import (
gorand "math/rand" gorand "math/rand"
"net/http" "net/http"
"os" "os"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/ecies"
"github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client" swarm "github.com/ethereum/go-ethereum/swarm/api/client"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/testutil"
) )
const (
hashRegexp = `[a-f\d]{128}`
data = "notsorandomdata"
)
var DefaultCurve = crypto.S256()
// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. // TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
// is then fetched through 2nd node. since the tested code is not key-aware - we can just // is then fetched through 2nd node. since the tested code is not key-aware - we can just
// fetch from the 2nd node using HTTP BasicAuth // fetch from the 2nd node using HTTP BasicAuth
func TestAccessPassword(t *testing.T) { func TestAccessPassword(t *testing.T) {
cluster := newTestCluster(t, 1) srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer cluster.Shutdown() defer srv.Close()
proxyNode := cluster.Nodes[0]
// create a tmp file dataFilename := testutil.TempFileWithContent(t, data)
tmp, err := ioutil.TempDir("", "swarm-test") defer os.RemoveAll(dataFilename)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
// write data to file
data := "notsorandomdata"
dataFilename := filepath.Join(tmp, "data.txt")
err = ioutil.WriteFile(dataFilename, []byte(data), 0666)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{128}`
// upload the file with 'swarm up' and expect a hash // upload the file with 'swarm up' and expect a hash
up := runSwarm(t, up := runSwarm(t,
"--bzzapi", "--bzzapi",
proxyNode.URL, //it doesn't matter through which node we upload content srv.URL, //it doesn't matter through which node we upload content
"up", "up",
"--encrypt", "--encrypt",
dataFilename) dataFilename)
@ -83,14 +76,14 @@ func TestAccessPassword(t *testing.T) {
} }
ref := matches[0] ref := matches[0]
tmp, err := ioutil.TempDir("", "swarm-test")
password := "smth"
passwordFilename := filepath.Join(tmp, "password.txt")
err = ioutil.WriteFile(passwordFilename, []byte(password), 0666)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(tmp)
password := "smth"
passwordFilename := testutil.TempFileWithContent(t, "smth")
defer os.RemoveAll(passwordFilename)
up = runSwarm(t, up = runSwarm(t,
"access", "access",
@ -142,8 +135,10 @@ func TestAccessPassword(t *testing.T) {
if a.KdfParams == nil { if a.KdfParams == nil {
t.Fatal("manifest access kdf params is nil") t.Fatal("manifest access kdf params is nil")
} }
if a.Publisher != "" {
client := swarm.NewClient(cluster.Nodes[0].URL) t.Fatal("should be empty")
}
client := swarm.NewClient(srv.URL)
hash, err := client.UploadManifest(&m, false) hash, err := client.UploadManifest(&m, false)
if err != nil { if err != nil {
@ -152,7 +147,7 @@ func TestAccessPassword(t *testing.T) {
httpClient := &http.Client{} httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash url := srv.URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url) response, err := httpClient.Get(url)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -188,17 +183,13 @@ func TestAccessPassword(t *testing.T) {
t.Errorf("expected decrypted data %q, got %q", data, string(d)) t.Errorf("expected decrypted data %q, got %q", data, string(d))
} }
wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt") wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng")
defer os.RemoveAll(wrongPasswordFilename)
err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666)
if err != nil {
t.Fatal(err)
}
//download file with 'swarm down' with wrong password //download file with 'swarm down' with wrong password
up = runSwarm(t, up = runSwarm(t,
"--bzzapi", "--bzzapi",
proxyNode.URL, srv.URL,
"down", "down",
"bzz:/"+hash, "bzz:/"+hash,
tmp, tmp,
@ -219,25 +210,11 @@ func TestAccessPassword(t *testing.T) {
// the test will fail if the proxy's given private key is not granted on the ACT. // the test will fail if the proxy's given private key is not granted on the ACT.
func TestAccessPK(t *testing.T) { func TestAccessPK(t *testing.T) {
// Setup Swarm and upload a test file to it // Setup Swarm and upload a test file to it
cluster := newTestCluster(t, 1) cluster := newTestCluster(t, 2)
defer cluster.Shutdown() defer cluster.Shutdown()
// create a tmp file dataFilename := testutil.TempFileWithContent(t, data)
tmp, err := ioutil.TempFile("", "swarm-test") defer os.RemoveAll(dataFilename)
if err != nil {
t.Fatal(err)
}
defer tmp.Close()
defer os.Remove(tmp.Name())
// write data to file
data := "notsorandomdata"
_, err = io.WriteString(tmp, data)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{128}`
// upload the file with 'swarm up' and expect a hash // upload the file with 'swarm up' and expect a hash
up := runSwarm(t, up := runSwarm(t,
@ -245,7 +222,7 @@ func TestAccessPK(t *testing.T) {
cluster.Nodes[0].URL, cluster.Nodes[0].URL,
"up", "up",
"--encrypt", "--encrypt",
tmp.Name()) dataFilename)
_, matches := up.ExpectRegexp(hashRegexp) _, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit() up.ExpectExit()
@ -254,7 +231,6 @@ func TestAccessPK(t *testing.T) {
} }
ref := matches[0] ref := matches[0]
pk := cluster.Nodes[0].PrivateKey pk := cluster.Nodes[0].PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey) granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
@ -263,22 +239,15 @@ func TestAccessPK(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
passFile, err := ioutil.TempFile("", "swarm-test") passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
if err != nil { defer os.RemoveAll(passwordFilename)
t.Fatal(err)
}
defer passFile.Close()
defer os.Remove(passFile.Name())
_, err = io.WriteString(passFile, testPassphrase)
if err != nil {
t.Fatal(err)
}
_, publisherAccount := getTestAccount(t, publisherDir) _, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t, up = runSwarm(t,
"--bzzaccount", "--bzzaccount",
publisherAccount.Address.String(), publisherAccount.Address.String(),
"--password", "--password",
passFile.Name(), passwordFilename,
"--datadir", "--datadir",
publisherDir, publisherDir,
"--bzzapi", "--bzzapi",
@ -299,6 +268,20 @@ func TestAccessPK(t *testing.T) {
t.Fatalf("stdout not matched") t.Fatalf("stdout not matched")
} }
//get the public key from the publisher directory
publicKeyFromDataDir := runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"print-keys",
"--compressed",
)
_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
publicKeyFromDataDir.ExpectExit()
pkComp := strings.Split(publicKeyString[0], "=")[1]
var m api.Manifest var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m) err = json.Unmarshal([]byte(matches[0]), &m)
@ -332,7 +315,9 @@ func TestAccessPK(t *testing.T) {
if a.KdfParams != nil { if a.KdfParams != nil {
t.Fatal("manifest access kdf params should be nil") t.Fatal("manifest access kdf params should be nil")
} }
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
client := swarm.NewClient(cluster.Nodes[0].URL) client := swarm.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false) hash, err := client.UploadManifest(&m, false)
@ -359,36 +344,34 @@ func TestAccessPK(t *testing.T) {
} }
} }
// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control // TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through
// one of the nodes then disappears.
func TestAccessACT(t *testing.T) { func TestAccessACT(t *testing.T) {
testAccessACT(t, 0)
}
// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
func TestAccessACTScale(t *testing.T) {
testAccessACT(t, 1000)
}
// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
// the publisher uploads through one of the nodes then disappears.
func testAccessACT(t *testing.T, bogusEntries int) {
// Setup Swarm and upload a test file to it // Setup Swarm and upload a test file to it
cluster := newTestCluster(t, 3) const clusterSize = 3
cluster := newTestCluster(t, clusterSize)
defer cluster.Shutdown() defer cluster.Shutdown()
var uploadThroughNode = cluster.Nodes[0] var uploadThroughNode = cluster.Nodes[0]
client := swarm.NewClient(uploadThroughNode.URL) client := swarm.NewClient(uploadThroughNode.URL)
r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`) nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)
// create a tmp file dataFilename := testutil.TempFileWithContent(t, data)
tmp, err := ioutil.TempFile("", "swarm-test") defer os.RemoveAll(dataFilename)
if err != nil {
t.Fatal(err)
}
defer tmp.Close()
defer os.Remove(tmp.Name())
// write data to file
data := "notsorandomdata"
_, err = io.WriteString(tmp, data)
if err != nil {
t.Fatal(err)
}
hashRegexp := `[a-f\d]{128}`
// upload the file with 'swarm up' and expect a hash // upload the file with 'swarm up' and expect a hash
up := runSwarm(t, up := runSwarm(t,
@ -396,7 +379,7 @@ func TestAccessACT(t *testing.T) {
cluster.Nodes[0].URL, cluster.Nodes[0].URL,
"up", "up",
"--encrypt", "--encrypt",
tmp.Name()) dataFilename)
_, matches := up.ExpectRegexp(hashRegexp) _, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit() up.ExpectExit()
@ -415,41 +398,42 @@ func TestAccessACT(t *testing.T) {
grantees = append(grantees, hex.EncodeToString(granteePubKey)) grantees = append(grantees, hex.EncodeToString(granteePubKey))
} }
granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv") if bogusEntries > 0 {
if err != nil { bogusGrantees := []string{}
t.Fatal(err)
}
_, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) for i := 0; i < bogusEntries; i++ {
if err != nil { prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)
t.Fatal(err) if err != nil {
t.Fatal(err)
}
bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey)))
}
r2 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
for i := 0; i < len(grantees); i++ {
insertAtIdx := r2.Intn(len(bogusGrantees))
bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...)
}
grantees = bogusGrantees
} }
granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n"))
defer granteesPubkeyListFile.Close() defer os.RemoveAll(granteesPubkeyListFile)
defer os.Remove(granteesPubkeyListFile.Name())
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(publisherDir)
passFile, err := ioutil.TempFile("", "swarm-test") passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
if err != nil { defer os.RemoveAll(passwordFilename)
t.Fatal(err) actPasswordFilename := testutil.TempFileWithContent(t, "smth")
} defer os.RemoveAll(actPasswordFilename)
defer passFile.Close()
defer os.Remove(passFile.Name())
_, err = io.WriteString(passFile, testPassphrase)
if err != nil {
t.Fatal(err)
}
_, publisherAccount := getTestAccount(t, publisherDir) _, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t, up = runSwarm(t,
"--bzzaccount", "--bzzaccount",
publisherAccount.Address.String(), publisherAccount.Address.String(),
"--password", "--password",
passFile.Name(), passwordFilename,
"--datadir", "--datadir",
publisherDir, publisherDir,
"--bzzapi", "--bzzapi",
@ -458,7 +442,9 @@ func TestAccessACT(t *testing.T) {
"new", "new",
"act", "act",
"--grant-keys", "--grant-keys",
granteesPubkeyListFile.Name(), granteesPubkeyListFile,
"--password",
actPasswordFilename,
ref, ref,
) )
@ -468,6 +454,22 @@ func TestAccessACT(t *testing.T) {
if len(matches) == 0 { if len(matches) == 0 {
t.Fatalf("stdout not matched") t.Fatalf("stdout not matched")
} }
//get the public key from the publisher directory
publicKeyFromDataDir := runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"print-keys",
"--compressed",
)
_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
publicKeyFromDataDir.ExpectExit()
pkComp := strings.Split(publicKeyString[0], "=")[1]
hash := matches[0] hash := matches[0]
m, _, err := client.DownloadManifest(hash) m, _, err := client.DownloadManifest(hash)
if err != nil { if err != nil {
@ -497,10 +499,10 @@ func TestAccessACT(t *testing.T) {
if len(a.Salt) < 32 { if len(a.Salt) < 32 {
t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
} }
if a.KdfParams != nil {
t.Fatal("manifest access kdf params should be nil")
}
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
httpClient := &http.Client{} httpClient := &http.Client{}
// all nodes except the skipped node should be able to decrypt the content // all nodes except the skipped node should be able to decrypt the content
@ -521,6 +523,25 @@ func TestAccessACT(t *testing.T) {
t.Fatalf("should be a 401") t.Fatalf("should be a 401")
} }
// try downloading using a password instead, using the unauthorized node
passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1)
response, err = httpClient.Get(passwordUrl)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
// now try with the wrong password, expect 401
passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1)
response, err = httpClient.Get(passwordUrl)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusUnauthorized {
t.Fatal("should be a 401")
}
continue continue
} }

View File

@ -59,27 +59,29 @@ var (
//constants for environment variables //constants for environment variables
const ( const (
SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT"
SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR"
SWARM_ENV_PORT = "SWARM_PORT" SWARM_ENV_PORT = "SWARM_PORT"
SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID"
SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE"
SWARM_ENV_SWAP_API = "SWARM_SWAP_API" SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE" SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE"
SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY" SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY"
SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE" SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK" SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE"
SWARM_ENV_ENS_API = "SWARM_ENS_API" SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_CORS = "SWARM_CORS" SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" SWARM_ENV_CORS = "SWARM_CORS"
SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE" SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES"
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
GETH_ENV_DATADIR = "GETH_DATADIR" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD"
SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH"
GETH_ENV_DATADIR = "GETH_DATADIR"
) )
// These settings ensure that TOML keys use the same names as Go struct fields. // These settings ensure that TOML keys use the same names as Go struct fields.
@ -124,7 +126,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
//get the account for the provided swarm account //get the account for the provided swarm account
prvkey := getAccount(config.BzzAccount, ctx, stack) prvkey := getAccount(config.BzzAccount, ctx, stack)
//set the resolved config path (geth --datadir) //set the resolved config path (geth --datadir)
config.Path = stack.InstanceDir() config.Path = expandPath(stack.InstanceDir())
//finally, initialize the configuration //finally, initialize the configuration
config.Init(prvkey) config.Init(prvkey)
//configuration phase completed here //configuration phase completed here
@ -175,14 +177,18 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
} }
if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" { if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
if id, _ := strconv.Atoi(networkid); id != 0 { id, err := strconv.ParseUint(networkid, 10, 64)
currentConfig.NetworkID = uint64(id) if err != nil {
utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err)
}
if id != 0 {
currentConfig.NetworkID = id
} }
} }
if ctx.GlobalIsSet(utils.DataDirFlag.Name) { if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" { if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" {
currentConfig.Path = datadir currentConfig.Path = expandPath(datadir)
} }
} }
@ -207,6 +213,9 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.SyncUpdateDelay = d currentConfig.SyncUpdateDelay = d
} }
// any value including 0 is acceptable
currentConfig.MaxStreamPeerServers = ctx.GlobalInt(SwarmMaxStreamPeerServersFlag.Name)
if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) { if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
currentConfig.LightNodeEnabled = true currentConfig.LightNodeEnabled = true
} }
@ -226,6 +235,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
if len(ensAPIs) == 1 && ensAPIs[0] == "" { if len(ensAPIs) == 1 && ensAPIs[0] == "" {
ensAPIs = nil ensAPIs = nil
} }
for i := range ensAPIs {
ensAPIs[i] = expandPath(ensAPIs[i])
}
currentConfig.EnsAPIs = ensAPIs currentConfig.EnsAPIs = ensAPIs
} }
@ -262,13 +275,17 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
} }
if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" { if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
if id, _ := strconv.Atoi(networkid); id != 0 { id, err := strconv.ParseUint(networkid, 10, 64)
currentConfig.NetworkID = uint64(id) if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
}
if id != 0 {
currentConfig.NetworkID = id
} }
} }
if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" { if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
currentConfig.Path = datadir currentConfig.Path = expandPath(datadir)
} }
bzzport := os.Getenv(SWARM_ENV_PORT) bzzport := os.Getenv(SWARM_ENV_PORT)
@ -281,33 +298,50 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
} }
if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" { if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
if swap, err := strconv.ParseBool(swapenable); err != nil { swap, err := strconv.ParseBool(swapenable)
currentConfig.SwapEnabled = swap if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
} }
currentConfig.SwapEnabled = swap
} }
if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" { if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
if sync, err := strconv.ParseBool(syncdisable); err != nil { sync, err := strconv.ParseBool(syncdisable)
currentConfig.SyncEnabled = !sync if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
} }
currentConfig.SyncEnabled = !sync
} }
if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" { if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
if skipCheck, err := strconv.ParseBool(v); err != nil { skipCheck, err := strconv.ParseBool(v)
if err != nil {
currentConfig.DeliverySkipCheck = skipCheck currentConfig.DeliverySkipCheck = skipCheck
} }
} }
if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" { if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
if d, err := time.ParseDuration(v); err != nil { d, err := time.ParseDuration(v)
currentConfig.SyncUpdateDelay = d if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
} }
currentConfig.SyncUpdateDelay = d
}
if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
m, err := strconv.Atoi(max)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
}
currentConfig.MaxStreamPeerServers = m
} }
if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" { if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
if lightnode, err := strconv.ParseBool(lne); err != nil { lightnode, err := strconv.ParseBool(lne)
currentConfig.LightNodeEnabled = lightnode if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
} }
currentConfig.LightNodeEnabled = lightnode
} }
if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" { if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {

View File

@ -29,6 +29,48 @@ import (
"gopkg.in/urfave/cli.v1" "gopkg.in/urfave/cli.v1"
) )
var dbCommand = cli.Command{
Name: "db",
CustomHelpTemplate: helpTemplate,
Usage: "manage the local chunk database",
ArgsUsage: "db COMMAND",
Description: "Manage the local chunk database",
Subcommands: []cli.Command{
{
Action: dbExport,
CustomHelpTemplate: helpTemplate,
Name: "export",
Usage: "export a local chunk database as a tar archive (use - to send to stdout)",
ArgsUsage: "<chunkdb> <file>",
Description: `
Export a local chunk database as a tar archive (use - to send to stdout).
swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
The export may be quite large, consider piping the output through the Unix
pv(1) tool to get a progress bar:
swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
`,
},
{
Action: dbImport,
CustomHelpTemplate: helpTemplate,
Name: "import",
Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
ArgsUsage: "<chunkdb> <file>",
Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
The import may be quite large, consider piping the input through the Unix
pv(1) tool to get a progress bar:
pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
},
},
}
func dbExport(ctx *cli.Context) { func dbExport(ctx *cli.Context) {
args := ctx.Args() args := ctx.Args()
if len(args) != 3 { if len(args) != 3 {
@ -93,21 +135,6 @@ func dbImport(ctx *cli.Context) {
log.Info(fmt.Sprintf("successfully imported %d chunks", count)) log.Info(fmt.Sprintf("successfully imported %d chunks", count))
} }
func dbClean(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
}
store, err := openLDBStore(args[0], common.Hex2Bytes(args[1]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
defer store.Close()
store.Cleanup()
}
func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) { func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
return nil, fmt.Errorf("invalid chunkdb path: %s", err) return nil, fmt.Errorf("invalid chunkdb path: %s", err)

View File

@ -28,6 +28,15 @@ import (
"gopkg.in/urfave/cli.v1" "gopkg.in/urfave/cli.v1"
) )
var downloadCommand = cli.Command{
Action: download,
Name: "down",
Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
Usage: "downloads a swarm manifest or a file inside a manifest",
ArgsUsage: " <uri> [<dir>]",
Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
}
func download(ctx *cli.Context) { func download(ctx *cli.Context) {
log.Debug("downloading content using swarm down") log.Debug("downloading content using swarm down")
args := ctx.Args() args := ctx.Args()

View File

@ -19,15 +19,15 @@ package main
import ( import (
"bytes" "bytes"
"crypto/md5" "crypto/md5"
"crypto/rand"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"os" "os"
"runtime"
"strings" "strings"
"testing" "testing"
"github.com/ethereum/go-ethereum/swarm" "github.com/ethereum/go-ethereum/swarm"
"github.com/ethereum/go-ethereum/swarm/testutil"
) )
// TestCLISwarmExportImport perform the following test: // TestCLISwarmExportImport perform the following test:
@ -38,14 +38,18 @@ import (
// 5. imports the exported datastore // 5. imports the exported datastore
// 6. fetches the uploaded random file from the second node // 6. fetches the uploaded random file from the second node
func TestCLISwarmExportImport(t *testing.T) { func TestCLISwarmExportImport(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
cluster := newTestCluster(t, 1) cluster := newTestCluster(t, 1)
// generate random 10mb file // generate random 10mb file
f, cleanup := generateRandomFile(t, 10000000) content := testutil.RandomBytes(1, 10000000)
defer cleanup() fileName := testutil.TempFileWithContent(t, string(content))
defer os.Remove(fileName)
// upload the file with 'swarm up' and expect a hash // upload the file with 'swarm up' and expect a hash
up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name()) up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
_, matches := up.ExpectRegexp(`[a-f\d]{64}`) _, matches := up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit() up.ExpectExit()
hash := matches[0] hash := matches[0]
@ -92,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
} }
// compare downloaded file with the generated random file // compare downloaded file with the generated random file
mustEqualFiles(t, f, res.Body) mustEqualFiles(t, bytes.NewReader(content), res.Body)
} }
func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) { func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@ -113,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen) t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
} }
} }
func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
// create a tmp file
tmp, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
// callback for tmp file cleanup
teardown = func() {
tmp.Close()
os.Remove(tmp.Name())
}
// write 10mb random data to file
buf := make([]byte, 10000000)
_, err = rand.Read(buf)
if err != nil {
t.Fatal(err)
}
ioutil.WriteFile(tmp.Name(), buf, 0755)
return tmp, teardown
}

Some files were not shown because too many files have changed in this diff Show More