Vdb 269- fetch logs by hash (#122)

* Upgrade geth from 1.8.15 to 1.8.18

* Update vat_tune to use shared repository methods

* Query blockchain by block hash instead of block number range

* Remove hash validation from repositories

* Fix vow flog integration test

* Update README Travis build sticker

* Update constants formatting per go fmt

* Update EthPublicKeyParser.ParsePublicKey to use discv5.PubkeyID method

* Address PR comments

Gopkg.lock

@@ -3,30 +3,23 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:a313376bcbcce8ae8bddb8089a7293e0473a0f8e9e3710d6244e09e81875ccf0"
   name = "github.com/aristanetworks/goarista"
   packages = ["monotime"]
-  pruneopts = ""
   revision = "ff33da284e760fcdb03c33d37a719e5ed30ba844"
 
 [[projects]]
   branch = "master"
-  digest = "1:c6bf1ac7bbc0fe51637bf54d5a88ff79b171b3b42dbc665dec98303c862d8662"
   name = "github.com/btcsuite/btcd"
   packages = ["btcec"]
-  pruneopts = ""
   revision = "cff30e1d23fc9e800b2b5b4b41ef1817dda07e9f"
 
 [[projects]]
-  digest = "1:aaeffbff5bd24654cb4c190ed75d6c7b57b4f5d6741914c1a7a6bb7447e756c5"
   name = "github.com/deckarep/golang-set"
   packages = ["."]
-  pruneopts = ""
   revision = "cbaa98ba5575e67703b32b4b19f73c91f3c4159e"
   version = "v1.7.1"
 
 [[projects]]
-  digest = "1:8acdd83e35b407d026e546bf4d18cc778e9604433d03d468eccca62b9c51c718"
   name = "github.com/ethereum/go-ethereum"
   packages = [
     ".",
@@ -35,9 +28,11 @@
     "accounts/abi/bind",
     "accounts/keystore",
     "common",
+    "common/bitutil",
     "common/hexutil",
     "common/math",
     "common/mclock",
+    "common/prque",
     "core/rawdb",
     "core/types",
     "crypto",
@@ -52,59 +47,49 @@
     "p2p",
     "p2p/discover",
     "p2p/discv5",
+    "p2p/enode",
+    "p2p/enr",
     "p2p/nat",
     "p2p/netutil",
     "params",
     "rlp",
     "rpc",
-    "trie",
+    "trie"
   ]
-  pruneopts = ""
+  revision = "58632d44021bf095b43a1bb2443e6e3690a94739"
-  revision = "89451f7c382ad2185987ee369f16416f89c28a7d"
+  version = "v1.8.18"
-  version = "v1.8.15"
 
 [[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
   name = "github.com/fsnotify/fsnotify"
   packages = ["."]
-  pruneopts = ""
   revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
   version = "v1.4.7"
 
 [[projects]]
-  digest = "1:a01080d20c45c031c13f3828c56e58f4f51d926a482ad10cc0316225097eb7ea"
   name = "github.com/go-stack/stack"
   packages = ["."]
-  pruneopts = ""
   revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a"
   version = "v1.8.0"
 
 [[projects]]
-  digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
   name = "github.com/golang/protobuf"
   packages = ["proto"]
-  pruneopts = ""
   revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
   version = "v1.2.0"
 
 [[projects]]
   branch = "master"
-  digest = "1:2a5888946cdbc8aa360fd43301f9fc7869d663f60d5eedae7d4e6e5e4f06f2bf"
   name = "github.com/golang/snappy"
   packages = ["."]
-  pruneopts = ""
   revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a"
 
 [[projects]]
-  digest = "1:5247b135b5492aa232a731acdcb52b08f32b874cb398f21ab460396eadbe866b"
   name = "github.com/google/uuid"
   packages = ["."]
-  pruneopts = ""
   revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494"
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec"
   name = "github.com/hashicorp/hcl"
   packages = [
     ".",
@@ -116,29 +101,25 @@
     "hcl/token",
     "json/parser",
     "json/scanner",
-    "json/token",
+    "json/token"
   ]
-  pruneopts = ""
   revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241"
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:b3c5b95e56c06f5aa72cb2500e6ee5f44fcd122872d4fec2023a488e561218bc"
   name = "github.com/hpcloud/tail"
   packages = [
     ".",
     "ratelimiter",
     "util",
     "watch",
-    "winfile",
+    "winfile"
   ]
-  pruneopts = ""
   revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5"
   version = "v1.0.0"
 
 [[projects]]
   branch = "master"
-  digest = "1:b6e4cc26365c004808649862e22069de09594a9222143399a7a04904e9f7018c"
   name = "github.com/huin/goupnp"
   packages = [
     ".",
@@ -147,75 +128,59 @@
     "httpu",
     "scpd",
     "soap",
-    "ssdp",
+    "ssdp"
   ]
-  pruneopts = ""
   revision = "1395d1447324cbea88d249fbfcfd70ea878fdfca"
 
 [[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
   name = "github.com/inconshreveable/mousetrap"
   packages = ["."]
-  pruneopts = ""
   revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
   version = "v1.0"
 
 [[projects]]
-  digest = "1:76f836364ae83ed811c415aa92e1209ce49de9f62aad85b85fca749a8b96a110"
   name = "github.com/jackpal/go-nat-pmp"
   packages = ["."]
-  pruneopts = ""
   revision = "c9cfead9f2a36ddf3daa40ba269aa7f4bbba6b62"
   version = "v1.0.1"
 
 [[projects]]
   branch = "master"
-  digest = "1:617ee2434b77e911fa26b678730be9a617f75243b194eadc8201c8ac860844aa"
   name = "github.com/jmoiron/sqlx"
   packages = [
     ".",
-    "reflectx",
+    "reflectx"
   ]
-  pruneopts = ""
   revision = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"
 
 [[projects]]
   branch = "master"
-  digest = "1:29145d7af4adafd72a79df5e41456ac9e232d5a28c1cd4dacf3ff008a217fc10"
   name = "github.com/lib/pq"
   packages = [
     ".",
-    "oid",
+    "oid"
   ]
-  pruneopts = ""
   revision = "4ded0e9383f75c197b3a2aaa6d590ac52df6fd79"
 
 [[projects]]
-  digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5"
   name = "github.com/magiconair/properties"
   packages = ["."]
-  pruneopts = ""
   revision = "c2353362d570a7bfa228149c62842019201cfb71"
   version = "v1.8.0"
 
 [[projects]]
-  digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf"
   name = "github.com/mitchellh/go-homedir"
   packages = ["."]
-  pruneopts = ""
   revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4"
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:5219b4506253ccc598f9340677162a42d6a78f340a4cc6df2d62db4d0593c4e9"
   name = "github.com/mitchellh/mapstructure"
   packages = ["."]
-  pruneopts = ""
   revision = "fa473d140ef3c6adf42d6b391fe76707f1f243c8"
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:a7fd918fb5bd2188436785c0424f8a50b4addfedf37a2b14d796be2a927b8007"
   name = "github.com/onsi/ginkgo"
   packages = [
     ".",
@@ -235,14 +200,12 @@
     "reporters/stenographer",
     "reporters/stenographer/support/go-colorable",
     "reporters/stenographer/support/go-isatty",
-    "types",
+    "types"
   ]
-  pruneopts = ""
   revision = "3774a09d95489ccaa16032e0770d08ea77ba6184"
   version = "v1.6.0"
 
 [[projects]]
-  digest = "1:3ecd0a37c4a90c12a97e31c398cdbc173824351aa891898ee178120bfe71c478"
   name = "github.com/onsi/gomega"
   packages = [
     ".",
@@ -257,106 +220,82 @@
     "matchers/support/goraph/edge",
     "matchers/support/goraph/node",
     "matchers/support/goraph/util",
-    "types",
+    "types"
   ]
-  pruneopts = ""
   revision = "7615b9433f86a8bdf29709bf288bc4fd0636a369"
   version = "v1.4.2"
 
 [[projects]]
-  digest = "1:a5484d4fa43127138ae6e7b2299a6a52ae006c7f803d98d717f60abf3e97192e"
   name = "github.com/pborman/uuid"
   packages = ["."]
-  pruneopts = ""
   revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
   version = "v1.2"
 
 [[projects]]
-  digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457"
   name = "github.com/pelletier/go-toml"
   packages = ["."]
-  pruneopts = ""
   revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
   version = "v1.2.0"
 
 [[projects]]
-  digest = "1:210286d0cb60ffe28f1ca00b664029e8943009f95d06d8f8c336301b28e1aee5"
   name = "github.com/philhofer/fwd"
   packages = ["."]
-  pruneopts = ""
   revision = "bb6d471dc95d4fe11e432687f8b70ff496cf3136"
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:7143292549152d009ca9e9c493b74736a2ebd93f921bea8a4b308d7cc5edc6b3"
   name = "github.com/rjeczalik/notify"
   packages = ["."]
-  pruneopts = ""
   revision = "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8"
   version = "v0.9.1"
 
 [[projects]]
-  digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf"
   name = "github.com/rs/cors"
   packages = ["."]
-  pruneopts = ""
   revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f"
   version = "v1.5.0"
 
 [[projects]]
-  digest = "1:d0431c2fd72e39ee43ea7742322abbc200c3e704c9102c5c3c2e2e667095b0ca"
   name = "github.com/spf13/afero"
   packages = [
     ".",
-    "mem",
+    "mem"
   ]
-  pruneopts = ""
   revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd"
   version = "v1.1.2"
 
 [[projects]]
-  digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
   name = "github.com/spf13/cast"
   packages = ["."]
-  pruneopts = ""
   revision = "8965335b8c7107321228e3e3702cab9832751bac"
   version = "v1.2.0"
 
 [[projects]]
-  digest = "1:a1403cc8a94b8d7956ee5e9694badef0e7b051af289caad1cf668331e3ffa4f6"
   name = "github.com/spf13/cobra"
   packages = ["."]
-  pruneopts = ""
   revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
   version = "v0.0.3"
 
 [[projects]]
-  digest = "1:9ceffa4ab5f7195ecf18b3a7fff90c837a9ed5e22e66d18069e4bccfe1f52aa0"
   name = "github.com/spf13/jwalterweatherman"
   packages = ["."]
-  pruneopts = ""
   revision = "4a4406e478ca629068e7768fc33f3f044173c0a6"
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:0a52bcb568386d98f4894575d53ce3e456f56471de6897bb8b9de13c33d9340e"
   name = "github.com/spf13/pflag"
   packages = ["."]
-  pruneopts = ""
   revision = "9a97c102cda95a86cec2345a6f09f55a939babf5"
   version = "v1.0.2"
 
 [[projects]]
-  digest = "1:ac25ea6cc1156aca9611411274b4a0bdd83a623845df6985aab508253955cc66"
   name = "github.com/spf13/viper"
   packages = ["."]
-  pruneopts = ""
   revision = "8fb642006536c8d3760c99d4fa2389f5e2205631"
   version = "v1.2.0"
 
 [[projects]]
   branch = "master"
-  digest = "1:ce5194e5afac308cc34e500cab45b4ce88a0742d689e3cf7e37b607ad76bed2f"
   name = "github.com/syndtr/goleveldb"
   packages = [
     "leveldb",
@@ -370,65 +309,53 @@
     "leveldb/opt",
     "leveldb/storage",
     "leveldb/table",
-    "leveldb/util",
+    "leveldb/util"
   ]
-  pruneopts = ""
   revision = "ae2bd5eed72d46b28834ec3f60db3a3ebedd8dbd"
 
 [[projects]]
-  digest = "1:cae65c1f3471b1f456a9d8a160440f4824ad50c718a15da4144490fbc3b12e49"
   name = "github.com/tinylib/msgp"
   packages = ["msgp"]
-  pruneopts = ""
   revision = "b2b6a672cf1e5b90748f79b8b81fc8c5cf0571a1"
   version = "1.0.2"
 
 [[projects]]
   branch = "master"
-  digest = "1:61a86f0be8b466d6e3fbdabb155aaa4006137cb5e3fd3b949329d103fa0ceb0f"
   name = "golang.org/x/crypto"
   packages = [
     "pbkdf2",
-    "scrypt",
+    "scrypt"
   ]
-  pruneopts = ""
   revision = "0e37d006457bf46f9e6692014ba72ef82c33022c"
 
 [[projects]]
   branch = "master"
-  digest = "1:fbdbb6cf8db3278412c9425ad78b26bb8eb788181f26a3ffb3e4f216b314f86a"
   name = "golang.org/x/net"
   packages = [
     "context",
     "html",
     "html/atom",
     "html/charset",
-    "websocket",
+    "websocket"
   ]
-  pruneopts = ""
   revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2"
 
 [[projects]]
   branch = "master"
-  digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1"
   name = "golang.org/x/sync"
   packages = ["errgroup"]
-  pruneopts = ""
   revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
 
 [[projects]]
   branch = "master"
-  digest = "1:70d519d5cddeb60ceda2db88c24c340b1b2d7efb25ab54bacb38f57ea1998df7"
   name = "golang.org/x/sys"
   packages = [
     "unix",
-    "windows",
+    "windows"
   ]
-  pruneopts = ""
   revision = "d641721ec2dead6fe5ca284096fe4b1fcd49e427"
 
 [[projects]]
-  digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
   name = "golang.org/x/text"
   packages = [
     "encoding",
@@ -450,108 +377,50 @@
     "runes",
     "transform",
     "unicode/cldr",
-    "unicode/norm",
+    "unicode/norm"
   ]
-  pruneopts = ""
   revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
   version = "v0.3.0"
 
 [[projects]]
-  branch = "master"
-  digest = "1:88401f863a34cfd82173ef2ad015652c2c47c866affc424bfdd4d5fafad1dd53"
-  name = "golang.org/x/tools"
-  packages = [
-    "go/ast/astutil",
-    "imports",
-    "internal/fastwalk",
-  ]
-  pruneopts = ""
-  revision = "90fa682c2a6e6a37b3a1364ce2fe1d5e41af9d6d"
-
-[[projects]]
-  digest = "1:be560af8c72c59788a6748eae60b8e1fe136fefdf049b7dfb6f5b591ea337984"
   name = "gopkg.in/DataDog/dd-trace-go.v1"
   packages = [
     "ddtrace",
     "ddtrace/ext",
     "ddtrace/internal",
-    "ddtrace/tracer",
+    "ddtrace/tracer"
   ]
-  pruneopts = ""
   revision = "bcd20367df871708a36549e7fe36183ee5b4fc55"
   version = "v1.3.0"
 
 [[projects]]
-  digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd"
   name = "gopkg.in/fsnotify.v1"
   packages = ["."]
-  pruneopts = ""
   revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
   source = "gopkg.in/fsnotify/fsnotify.v1"
   version = "v1.4.7"
 
 [[projects]]
   branch = "v2"
-  digest = "1:a585c075875ab9c344f7840a927f09f3285563f7318e761a1d61d642316f2217"
-  name = "gopkg.in/karalabe/cookiejar.v2"
-  packages = ["collections/prque"]
-  pruneopts = ""
-  revision = "8dcd6a7f4951f6ff3ee9cbb919a06d8925822e57"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:4f830ee018eb8c56d0def653ad7c9a1d2a053f0cef2ac6b2200f73b98fa6a681"
   name = "gopkg.in/natefinch/npipe.v2"
   packages = ["."]
-  pruneopts = ""
   revision = "c1b8fa8bdccecb0b8db834ee0b92fdbcfa606dd6"
 
 [[projects]]
   branch = "v1"
-  digest = "1:a96d16bd088460f2e0685d46c39bcf1208ba46e0a977be2df49864ec7da447dd"
   name = "gopkg.in/tomb.v1"
   packages = ["."]
-  pruneopts = ""
   revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8"
 
 [[projects]]
-  digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
   name = "gopkg.in/yaml.v2"
   packages = ["."]
-  pruneopts = ""
   revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
   version = "v2.2.1"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  input-imports = [
+  inputs-digest = "2fbede45ac5daf4cab0ad947ea580193383fddfe0b5d1ab924bd61959839ba1c"
-    "github.com/ethereum/go-ethereum",
-    "github.com/ethereum/go-ethereum/accounts/abi",
-    "github.com/ethereum/go-ethereum/accounts/abi/bind",
-    "github.com/ethereum/go-ethereum/common",
-    "github.com/ethereum/go-ethereum/common/hexutil",
-    "github.com/ethereum/go-ethereum/core/rawdb",
-    "github.com/ethereum/go-ethereum/core/types",
-    "github.com/ethereum/go-ethereum/crypto",
-    "github.com/ethereum/go-ethereum/ethclient",
-    "github.com/ethereum/go-ethereum/ethdb",
-    "github.com/ethereum/go-ethereum/p2p",
-    "github.com/ethereum/go-ethereum/p2p/discover",
-    "github.com/ethereum/go-ethereum/params",
-    "github.com/ethereum/go-ethereum/rlp",
-    "github.com/ethereum/go-ethereum/rpc",
-    "github.com/jmoiron/sqlx",
-    "github.com/lib/pq",
-    "github.com/mitchellh/go-homedir",
-    "github.com/onsi/ginkgo",
-    "github.com/onsi/gomega",
-    "github.com/onsi/gomega/ghttp",
-    "github.com/spf13/cobra",
-    "github.com/spf13/viper",
-    "golang.org/x/net/context",
-    "golang.org/x/sync/errgroup",
-    "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer",
-  ]
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -47,4 +47,4 @@
 
 [[constraint]]
   name = "github.com/ethereum/go-ethereum"
-  version = "1.8.15"
+  version = "1.8.18"

README.md

@@ -2,7 +2,7 @@
 
 [![Join the chat at https://gitter.im/vulcanizeio/VulcanizeDB](https://badges.gitter.im/vulcanizeio/VulcanizeDB.svg)](https://gitter.im/vulcanizeio/VulcanizeDB?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-[![Build Status](https://travis-ci.org/vulcanize/vulcanizedb.svg?branch=master)](https://travis-ci.org/vulcanize/vulcanizedb)
+[![Build Status](https://travis-ci.com/8thlight/maker-vulcanizedb.svg?branch=staging)](https://travis-ci.com/8thlight/maker-vulcanizedb)
 
 ## About
 

pkg/crypto/parser.go

@@ -2,7 +2,7 @@ package crypto
 
 import (
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/discv5"
 )
 
 type PublicKeyParser interface {
@@ -16,6 +16,6 @@ func (EthPublicKeyParser) ParsePublicKey(privateKey string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	pubKey := discover.PubkeyID(&np.PublicKey)
+	pubKey := discv5.PubkeyID(&np.PublicKey)
 	return pubKey.String(), nil
 }

pkg/transformers/bite/repository.go

@@ -42,12 +42,6 @@ func (repository BiteRepository) Create(headerID int64, models []interface{}) er
 			return fmt.Errorf("model of type %T, not %T", model, BiteModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, biteModel.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.bite (header_id, ilk, urn, ink, art, iart, tab, nflip, log_idx, tx_idx, raw_log)
         VALUES($1, $2, $3, $4::NUMERIC, $5::NUMERIC, $6::NUMERIC, $7::NUMERIC, $8::NUMERIC, $9, $10, $11)`,

pkg/transformers/cat_file/chop_lump/repository.go

@@ -39,12 +39,6 @@ func (repository CatFileChopLumpRepository) Create(headerID int64, models []inte
 			return fmt.Errorf("model of type %T, not %T", model, CatFileChopLumpModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, chopLump.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.cat_file_chop_lump (header_id, ilk, what, data, tx_idx, log_idx, raw_log)
 			VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/cat_file/flip/repository.go

@@ -38,12 +38,6 @@ func (repository CatFileFlipRepository) Create(headerID int64, models []interfac
 			return fmt.Errorf("model of type %T, not %T", model, CatFileFlipModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, flip.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = repository.db.Exec(
 			`INSERT into maker.cat_file_flip (header_id, ilk, what, flip, tx_idx, log_idx, raw_log)
 			VALUES($1, $2, $3, $4, $5, $6, $7)`,

pkg/transformers/cat_file/pit_vow/repository.go

@@ -38,12 +38,6 @@ func (repository CatFilePitVowRepository) Create(headerID int64, models []interf
 			return fmt.Errorf("model of type %T, not %T", model, CatFilePitVowModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vow.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = repository.db.Exec(
 			`INSERT into maker.cat_file_pit_vow (header_id, what, data, tx_idx, log_idx, raw_log)
 			VALUES($1, $2, $3, $4, $5, $6)`,

pkg/transformers/deal/repository.go

@@ -39,12 +39,6 @@ func (repository DealRepository) Create(headerID int64, models []interface{}) er
 			return fmt.Errorf("model of type %T, not %T", model, DealModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, dealModel.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.deal (header_id, bid_id, contract_address, log_idx, tx_idx, raw_log)
 					 VALUES($1, $2, $3, $4, $5, $6)`,

pkg/transformers/dent/repository.go

@@ -44,12 +44,6 @@ func (repository DentRepository) Create(headerID int64, models []interface{}) er
 			return fmt.Errorf("model of type %T, not %T", model, DentModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, dent.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.dent (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
 			VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)`,

pkg/transformers/drip_drip/repository.go

@@ -38,12 +38,6 @@ func (repository DripDripRepository) Create(headerID int64, models []interface{}
 			return fmt.Errorf("model of type %T, not %T", model, DripDripModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, dripDrip.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.drip_drip (header_id, ilk, log_idx, tx_idx, raw_log)
         VALUES($1, $2, $3, $4, $5)`,

pkg/transformers/drip_file/ilk/repository.go

@@ -39,12 +39,6 @@ func (repository DripFileIlkRepository) Create(headerID int64, models []interfac
 			return fmt.Errorf("model of type %T, not %T", model, DripFileIlkModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, ilk.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.drip_file_ilk (header_id, ilk, vow, tax, log_idx, tx_idx, raw_log)
         	VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/drip_file/repo/repository.go

@@ -39,12 +39,6 @@ func (repository DripFileRepoRepository) Create(headerID int64, models []interfa
 			return fmt.Errorf("model of type %T, not %T", model, DripFileRepoModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, repo.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.drip_file_repo (header_id, what, data, log_idx, tx_idx, raw_log)
         	VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`,

pkg/transformers/drip_file/vow/repository.go

@@ -39,12 +39,6 @@ func (repository DripFileVowRepository) Create(headerID int64, models []interfac
 			return fmt.Errorf("model of type %T, not %T", model, DripFileVowModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vow.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.drip_file_vow (header_id, what, data, log_idx, tx_idx, raw_log)
         	VALUES($1, $2, $3, $4, $5, $6)`,

pkg/transformers/factories/log_note_transformer.go

@@ -52,7 +52,7 @@ func (transformer LogNoteTransformer) Execute() error {
 	log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
 	for _, header := range missingHeaders {
 		// Fetch the missing logs for a given header
-		matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header.BlockNumber)
+		matchingLogs, err := transformer.Fetcher.FetchLogs(transformer.Config.ContractAddresses, topic, header)
 		if err != nil {
 			log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
 			return err

pkg/transformers/factories/transformer.go

@@ -48,7 +48,7 @@ func (transformer Transformer) Execute() error {
 	}
 	log.Printf("Fetching %v event logs for %d headers \n", transformerName, len(missingHeaders))
 	for _, header := range missingHeaders {
-		logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header.BlockNumber)
+		logs, err := transformer.Fetcher.FetchLogs(config.ContractAddresses, topics, header)
 		if err != nil {
 			log.Printf("Error fetching matching logs in %v transformer: %v", transformerName, err)
 			return err

pkg/transformers/flap_kick/repository.go

@@ -37,12 +37,6 @@ func (repository *FlapKickRepository) Create(headerID int64, models []interface{
 			return fmt.Errorf("model of type %T, not %T", model, FlapKickModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, flapKickModel.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.flap_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log)
         VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`,

pkg/transformers/flip_kick/repository.go

@@ -38,12 +38,6 @@ func (repository FlipKickRepository) Create(headerID int64, models []interface{}
 			return fmt.Errorf("model of type %T, not %T", model, FlipKickModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, flipKickModel.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.flip_kick (header_id, bid_id, lot, bid, gal, "end", urn, tab, tx_idx, log_idx, raw_log)
         VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8::NUMERIC, $9, $10, $11)`,

pkg/transformers/flop_kick/repository.go

@@ -38,12 +38,6 @@ func (repository FlopKickRepository) Create(headerID int64, models []interface{}
 			return fmt.Errorf("model of type %T, not %T", flopKick, Model{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, flopKickModel.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.flop_kick (header_id, bid_id, lot, bid, gal, "end", tx_idx, log_idx, raw_log)
         VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`,

pkg/transformers/frob/repository.go

@@ -39,12 +39,6 @@ func (repository FrobRepository) Create(headerID int64, models []interface{}) er
 			return fmt.Errorf("model of type %T, not %T", model, FrobModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, frobModel.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(`INSERT INTO maker.frob (header_id, art, dart, dink, iart, ilk, ink, urn, raw_log, log_idx, tx_idx)
 		VALUES($1, $2::NUMERIC, $3::NUMERIC, $4::NUMERIC, $5::NUMERIC, $6, $7::NUMERIC, $8, $9, $10, $11)`,
 			headerID, frobModel.Art, frobModel.Dart, frobModel.Dink, frobModel.IArt, frobModel.Ilk, frobModel.Ink, frobModel.Urn, frobModel.Raw, frobModel.LogIndex, frobModel.TransactionIndex)

pkg/transformers/integration_tests/vow_flog.go

@@ -39,7 +39,7 @@ var _ = Describe("VowFlog LogNoteTransformer", func() {
 		db := test_config.NewTestDB(blockchain.Node())
 		test_config.CleanTestDB(db)
 
-		err = persistHeader(db, blockNumber)
+		err = persistHeader(db, blockNumber, blockchain)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(1).To(Equal(1))
 

pkg/transformers/pit_file/debt_ceiling/repository.go

@@ -39,12 +39,6 @@ func (repository PitFileDebtCeilingRepository) Create(headerID int64, models []i
 			return fmt.Errorf("model of type %T, not %T", model, PitFileDebtCeilingModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, pitFileDC.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.pit_file_debt_ceiling (header_id, what, data, log_idx, tx_idx, raw_log)
         VALUES($1, $2, $3::NUMERIC, $4, $5, $6)`,

pkg/transformers/pit_file/ilk/repository.go

@@ -39,12 +39,6 @@ func (repository PitFileIlkRepository) Create(headerID int64, models []interface
 			return fmt.Errorf("model of type %T, not %T", model, PitFileIlkModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, pitFileIlk.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.pit_file_ilk (header_id, ilk, what, data, log_idx, tx_idx, raw_log)
         VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/price_feeds/repository.go

@@ -38,12 +38,6 @@ func (repository PriceFeedRepository) Create(headerID int64, models []interface{
 			return fmt.Errorf("model of type %T, not %T", model, PriceFeedModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, priceUpdate.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(`INSERT INTO maker.price_feeds (block_number, header_id, medianizer_address, usd_value, log_idx, tx_idx, raw_log)
 		VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`, priceUpdate.BlockNumber, headerID, priceUpdate.MedianizerAddress, priceUpdate.UsdValue, priceUpdate.LogIndex, priceUpdate.TransactionIndex, priceUpdate.Raw)
 		if err != nil {

pkg/transformers/shared/log_fetcher.go

@@ -15,8 +15,6 @@
 package shared
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -25,7 +23,7 @@ import (
 )
 
 type LogFetcher interface {
-	FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error)
+	FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error)
 }
 
 type SettableLogFetcher interface {
@@ -47,12 +45,11 @@ func NewFetcher(blockchain core.BlockChain) Fetcher {
 	}
 }
 
-func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) {
+func (fetcher Fetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
-	block := big.NewInt(blockNumber)
 	addresses := hexStringsToAddresses(contractAddresses)
+	blockHash := common.HexToHash(header.Hash)
 	query := ethereum.FilterQuery{
-		FromBlock: block,
+		BlockHash: &blockHash,
-		ToBlock:   block,
 		Addresses: addresses,
 		Topics:    topics,
 	}

pkg/transformers/shared/log_fetcher_test.go

@@ -15,13 +15,12 @@
 package shared_test
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
@@ -31,18 +30,20 @@ var _ = Describe("Fetcher", func() {
 		It("fetches logs based on the given query", func() {
 			blockChain := fakes.NewMockBlockChain()
 			fetcher := shared.NewFetcher(blockChain)
-			blockNumber := int64(123)
+			header := fakes.FakeHeader
+
 			addresses := []string{"0xfakeAddress", "0xanotherFakeAddress"}
 			topicZeros := [][]common.Hash{{common.BytesToHash([]byte{1, 2, 3, 4, 5})}}
 
-			_, err := fetcher.FetchLogs(addresses, topicZeros, blockNumber)
+			_, err := fetcher.FetchLogs(addresses, topicZeros, header)
 
 			address1 := common.HexToAddress("0xfakeAddress")
 			address2 := common.HexToAddress("0xanotherFakeAddress")
 			Expect(err).NotTo(HaveOccurred())
+
+			blockHash := common.HexToHash(header.Hash)
 			expectedQuery := ethereum.FilterQuery{
-				FromBlock: big.NewInt(blockNumber),
+				BlockHash: &blockHash,
-				ToBlock:   big.NewInt(blockNumber),
 				Addresses: []common.Address{address1, address2},
 				Topics:    topicZeros,
 			}
@@ -54,7 +55,7 @@ var _ = Describe("Fetcher", func() {
 			blockChain.SetGetEthLogsWithCustomQueryErr(fakes.FakeError)
 			fetcher := shared.NewFetcher(blockChain)
 
-			_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, int64(1))
+			_, err := fetcher.FetchLogs([]string{}, [][]common.Hash{}, core.Header{})
 
 			Expect(err).To(HaveOccurred())
 			Expect(err).To(MatchError(fakes.FakeError))

pkg/transformers/shared/repository.go

@@ -2,41 +2,10 @@ package shared
 
 import (
 	"database/sql"
-	"encoding/json"
-	"errors"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-var ErrHeaderMismatch = errors.New("log's block hash does not match associated header's block hash")
-
-func ValidateHeaderConsistency(headerID int64, rawLog []byte, db *postgres.DB) error {
-	var log types.Log
-	err := json.Unmarshal(rawLog, &log)
-	if err != nil {
-		return err
-	}
-	var hash string
-	err = db.Get(&hash, `SELECT hash FROM public.headers WHERE id = $1`, headerID)
-	if err != nil {
-		return err
-	}
-	if hash != log.BlockHash.String() {
-		err = deleteHeader(headerID, db)
-		if err != nil {
-			return err
-		}
-		return ErrHeaderMismatch
-	}
-	return nil
-}
-
-func deleteHeader(headerID int64, db *postgres.DB) error {
-	_, err := db.Exec(`DELETE FROM public.headers WHERE id = $1`, headerID)
-	return err
-}
-
 func MarkHeaderChecked(headerID int64, db *postgres.DB, checkedHeadersColumn string) error {
 	_, err := db.Exec(`INSERT INTO public.checked_headers (header_id, `+checkedHeadersColumn+`)
 		VALUES ($1, $2) 
@@ -59,14 +28,14 @@ func MissingHeaders(startingBlockNumber, endingBlockNumber int64, db *postgres.D
 	var err error
 
 	if endingBlockNumber == -1 {
-		query = `SELECT headers.id, headers.block_number FROM headers
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
 				LEFT JOIN checked_headers on headers.id = header_id
 				WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
 				AND headers.block_number >= $1
 				AND headers.eth_node_fingerprint = $2`
 		err = db.Select(&result, query, startingBlockNumber, db.Node.ID)
 	} else {
-		query = `SELECT headers.id, headers.block_number FROM headers
+		query = `SELECT headers.id, headers.block_number, headers.hash FROM headers
 				LEFT JOIN checked_headers on headers.id = header_id
 				WHERE (header_id ISNULL OR ` + checkedHeadersColumn + ` IS FALSE)
 				AND headers.block_number >= $1

pkg/transformers/tend/repository.go

@@ -44,12 +44,6 @@ func (repository TendRepository) Create(headerID int64, models []interface{}) er
 			return fmt.Errorf("model of type %T, not %T", model, TendModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, tend.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.tend (header_id, bid_id, lot, bid, guy, tic, log_idx, tx_idx, raw_log)
 			VALUES($1, $2, $3::NUMERIC, $4::NUMERIC, $5, $6, $7, $8, $9)`,

pkg/transformers/test_data/mocks/log_fetcher.go

@@ -29,10 +29,10 @@ type MockLogFetcher struct {
 	SetBcCalled              bool
 }
 
-func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, blockNumber int64) ([]types.Log, error) {
+func (mlf *MockLogFetcher) FetchLogs(contractAddresses []string, topics [][]common.Hash, header core.Header) ([]types.Log, error) {
 	mlf.FetchedContractAddresses = append(mlf.FetchedContractAddresses, contractAddresses)
 	mlf.FetchedTopics = topics
-	mlf.FetchedBlocks = append(mlf.FetchedBlocks, blockNumber)
+	mlf.FetchedBlocks = append(mlf.FetchedBlocks, header.BlockNumber)
 
 	return mlf.FetchedLogs, mlf.fetcherError
 }

pkg/transformers/test_data/shared_behaviors/repository_behaviors.go

@@ -15,7 +15,6 @@
 package shared_behaviors
 
 import (
-	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -24,7 +23,6 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/factories"
-	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/test_data"
 	"github.com/vulcanize/vulcanizedb/test_config"
 	"math/rand"
@@ -103,35 +101,6 @@ func SharedRepositoryCreateBehaviors(inputs *CreateBehaviorInputs) {
 			Expect(err.Error()).To(ContainSubstring("pq: duplicate key value violates unique constraint"))
 		})
 
-		Describe("when log's block hash does not match associated header's block hash", func() {
-			var (
-				differentHeaderID int64
-				err               error
-			)
-
-			BeforeEach(func() {
-				differentHeader := fakes.FakeHeader
-				differentHeader.BlockNumber = fakes.FakeHeader.BlockNumber + 1
-				differentHeader.Hash = common.BytesToHash(append(fakes.FakeHash.Bytes(), 1)).String()
-				differentHeaderID, err = headerRepository.CreateOrUpdateHeader(differentHeader)
-				Expect(err).NotTo(HaveOccurred())
-
-				err = repository.Create(differentHeaderID, []interface{}{logEventModel})
-			})
-
-			It("returns error", func() {
-				Expect(err).To(HaveOccurred())
-				Expect(err).To(MatchError(shared.ErrHeaderMismatch))
-			})
-
-			It("removes header", func() {
-				var headerIDs []int64
-				err = db.Select(&headerIDs, `SELECT id FROM public.headers`)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(headerIDs).NotTo(ContainElement(differentHeaderID))
-			})
-		})
-
 		It("allows for multiple log events of the same type in one transaction if they have different log indexes", func() {
 			err = repository.Create(headerID, []interface{}{logEventModel})
 			Expect(err).NotTo(HaveOccurred())

pkg/transformers/vat_flux/repository.go

@@ -39,12 +39,6 @@ func (repository VatFluxRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatFluxModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatFlux.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(`INSERT INTO maker.vat_flux (header_id, ilk, dst, src, rad, tx_idx, log_idx, raw_log)
 		VALUES($1, $2, $3, $4, $5::NUMERIC, $6, $7, $8)`,
 			headerID, vatFlux.Ilk, vatFlux.Dst, vatFlux.Src, vatFlux.Rad, vatFlux.TransactionIndex, vatFlux.LogIndex, vatFlux.Raw)

pkg/transformers/vat_fold/repository.go

@@ -39,12 +39,6 @@ func (repository VatFoldRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatFoldModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatFold.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.vat_fold (header_id, ilk, urn, rate, log_idx, tx_idx, raw_log)
 				VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/vat_grab/repository.go

@@ -25,12 +25,6 @@ func (repository VatGrabRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatGrabModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatGrab.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.vat_grab (header_id, ilk, urn, v, w, dink, dart, log_idx, tx_idx, raw_log)
 	   VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`,

pkg/transformers/vat_heal/repository.go

@@ -44,12 +44,6 @@ func (repository VatHealRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatHealModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatHeal.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err := tx.Exec(`INSERT INTO maker.vat_heal (header_id, urn, v, rad, log_idx, tx_idx, raw_log)
 		VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,
 			headerID, vatHeal.Urn, vatHeal.V, vatHeal.Rad, vatHeal.LogIndex, vatHeal.TransactionIndex, vatHeal.Raw)

pkg/transformers/vat_init/repository.go

@@ -39,12 +39,6 @@ func (repository VatInitRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatInitModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatInit.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT INTO maker.vat_init (header_id, ilk, log_idx, tx_idx, raw_log)
 			VALUES($1, $2, $3, $4, $5)`,

pkg/transformers/vat_move/repository.go

@@ -39,12 +39,6 @@ func (repository VatMoveRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatMoveModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatMove.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT INTO maker.vat_move (header_id, src, dst, rad, log_idx, tx_idx, raw_log)
 				VALUES ($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/vat_slip/repository.go

@@ -24,12 +24,6 @@ func (repository VatSlipRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatSlipModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatSlip.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.vat_slip (header_id, ilk, guy, rad, tx_idx, log_idx, raw_log)
 			VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/vat_toll/repository.go

@@ -24,12 +24,6 @@ func (repository VatTollRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatTollModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatToll.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.vat_toll (header_id, ilk, urn, take, tx_idx, log_idx, raw_log)
 			VALUES($1, $2, $3, $4::NUMERIC, $5, $6, $7)`,

pkg/transformers/vat_tune/repository.go

@@ -5,6 +5,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
+	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared/constants"
 )
 
 type VatTuneRepository struct {
@@ -23,12 +24,6 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VatTuneModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, vatTune.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.vat_tune (header_id, ilk, urn, v, w, dink, dart, tx_idx, log_idx, raw_log)
 	   VALUES($1, $2, $3, $4, $5, $6::NUMERIC, $7::NUMERIC, $8, $9, $10)`,
@@ -40,10 +35,7 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
 		}
 	}
 
-	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked)
+	err = shared.MarkHeaderCheckedInTransaction(headerID, tx, constants.VatTuneChecked)
-		VALUES ($1, $2)
-	ON CONFLICT (header_id) DO
-		UPDATE SET vat_tune_checked = $2`, headerID, true)
 	if err != nil {
 		tx.Rollback()
 		return err
@@ -52,28 +44,11 @@ func (repository VatTuneRepository) Create(headerID int64, models []interface{})
 }
 
 func (repository VatTuneRepository) MarkHeaderChecked(headerID int64) error {
-	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, vat_tune_checked)
+	return shared.MarkHeaderChecked(headerID, repository.db, constants.VatTuneChecked)
-		VALUES ($1, $2)
-	ON CONFLICT (header_id) DO
-		UPDATE SET vat_tune_checked = $2`, headerID, true)
-	return err
 }
 
 func (repository VatTuneRepository) MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error) {
-	var result []core.Header
+	return shared.MissingHeaders(startingBlockNumber, endingBlockNumber, repository.db, constants.VatTuneChecked)
-	err := repository.db.Select(
-		&result,
-		`SELECT headers.id, headers.block_number FROM headers
-	          LEFT JOIN checked_headers on headers.id = header_id
-	          WHERE (header_id ISNULL OR vat_tune_checked IS FALSE)
-	          AND headers.block_number >= $1
-	          AND headers.block_number <= $2
-	          AND headers.eth_node_fingerprint = $3`,
-		startingBlockNumber,
-		endingBlockNumber,
-		repository.db.Node.ID,
-	)
-	return result, err
 }
 
 func (repository *VatTuneRepository) SetDB(db *postgres.DB) {

pkg/transformers/vow_flog/repository.go

@@ -39,12 +39,6 @@ func (repository VowFlogRepository) Create(headerID int64, models []interface{})
 			return fmt.Errorf("model of type %T, not %T", model, VowFlogModel{})
 		}
 
-		err = shared.ValidateHeaderConsistency(headerID, flog.Raw, repository.db)
-		if err != nil {
-			tx.Rollback()
-			return err
-		}
-
 		_, err = tx.Exec(
 			`INSERT into maker.vow_flog (header_id, era, log_idx, tx_idx, raw_log)
 			VALUES($1, $2::NUMERIC, $3, $4, $5)`,

vendor/github.com/ethereum/go-ethereum/.github/CODEOWNERS

@@ -9,24 +9,27 @@ les/                            @zsfelfoldi
 light/                          @zsfelfoldi
 mobile/                         @karalabe
 p2p/                            @fjl @zsfelfoldi
+p2p/simulations                 @lmars
+p2p/protocols                   @zelig
+swarm/api/http                  @justelad
 swarm/bmt                       @zelig
 swarm/dev                       @lmars
 swarm/fuse                      @jmozah @holisticode
 swarm/grafana_dashboards        @nonsense
 swarm/metrics                   @nonsense @holisticode
 swarm/multihash                 @nolash
-swarm/network/bitvector         @zelig @janos @gbalint
+swarm/network/bitvector         @zelig @janos 
-swarm/network/priorityqueue     @zelig @janos @gbalint
+swarm/network/priorityqueue     @zelig @janos 
-swarm/network/simulations       @zelig
+swarm/network/simulations       @zelig @janos
-swarm/network/stream            @janos @zelig @gbalint @holisticode @justelad
+swarm/network/stream            @janos @zelig @holisticode @justelad
 swarm/network/stream/intervals  @janos
 swarm/network/stream/testing    @zelig
 swarm/pot                       @zelig
 swarm/pss                       @nolash @zelig @nonsense
 swarm/services                  @zelig
 swarm/state                     @justelad
-swarm/storage/encryption        @gbalint @zelig @nagydani
+swarm/storage/encryption        @zelig @nagydani
 swarm/storage/mock              @janos
-swarm/storage/mru               @nolash
+swarm/storage/feed              @nolash @jpeletier
 swarm/testutil                  @lmars
 whisper/                        @gballet @gluk256

vendor/github.com/ethereum/go-ethereum/.github/CONTRIBUTING.md

@@ -1,16 +1,40 @@
+# Contributing
+
+Thank you for considering to help out with the source code! We welcome 
+contributions from anyone on the internet, and are grateful for even the 
+smallest of fixes!
+
+If you'd like to contribute to go-ethereum, please fork, fix, commit and send a 
+pull request for the maintainers to review and merge into the main code base. If
+you wish to submit more complex changes though, please check up with the core 
+devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to 
+ensure those changes are in line with the general philosophy of the project 
+and/or get some early feedback which can make both your efforts much lighter as
+well as our review and merge procedures quick and simple.
+
+## Coding guidelines
+
+Please make sure your contributions adhere to our coding guidelines:
+
+ * Code must adhere to the official Go 
+[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines 
+(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must be documented adhering to the official Go 
+[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
+ * Pull requests need to be based on and opened against the `master` branch.
+ * Commit messages should be prefixed with the package(s) they modify.
+   * E.g. "eth, rpc: make trace configs optional"
+
 ## Can I have feature X
 
-Before you do a feature request please check and make sure that it isn't possible
+Before you submit a feature request, please check and make sure that it isn't 
-through some other means. The JavaScript enabled console is a powerful feature
+possible through some other means. The JavaScript-enabled console is a powerful 
-in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+feature in the right hands. Please check our 
+[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
-## Contributing
+## Configuration, dependencies, and tests
 
-If you'd like to contribute to go-ethereum please fork, fix, commit and
+Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-send a pull request. Commits which do not comply with the coding standards
+for more details on configuring your environment, managing project dependencies
-are ignored (use gofmt!).
+and testing procedures.
-
-See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, testing, and
-dependency management.

vendor/github.com/ethereum/go-ethereum/.github/no-response.yml

@@ -7,5 +7,5 @@ closeComment: >
   This issue has been automatically closed because there has been no response
   to our request for more information from the original author. With only the
   information that is currently in the issue, we don't have enough information
-  to take action. Please reach out if you have or find the answers we need so
+  to take action. Please reach out if you have more relevant information or
-  that we can investigate further.
+  answers to our questions so that we can investigate further.

vendor/github.com/ethereum/go-ethereum/.travis.yml

@@ -6,7 +6,7 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.9.x
+      go: 1.10.x
       script:
       - sudo modprobe fuse
       - sudo chmod 666 /dev/fuse
@@ -18,7 +18,7 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.11.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
@@ -27,7 +27,7 @@ matrix:
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: 1.10.x
+      go: 1.11.x
       script:
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - go run build/ci.go install
@@ -36,7 +36,7 @@ matrix:
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - lint
       git:
@@ -45,9 +45,10 @@ matrix:
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA upload
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - ubuntu-ppa
       git:
@@ -63,10 +64,11 @@ matrix:
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
 
     # This builder does the Linux Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-linux
       git:
@@ -96,11 +98,12 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       services:
         - docker
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-linux-mips
       git:
@@ -123,7 +126,8 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       addons:
         apt:
@@ -144,7 +148,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.10.3.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -160,8 +164,9 @@ matrix:
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - os: osx
+    - if: type = push
-      go: 1.10.x
+      os: osx
+      go: 1.11.x
       env:
         - azure-osx
         - azure-ios
@@ -188,19 +193,13 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - os: linux
+    - if: type = cron
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-purge
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
-
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/e09ccdce1048c5e03445
-    on_success: change
-    on_failure: always

vendor/github.com/ethereum/go-ethereum/Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.11-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

vendor/github.com/ethereum/go-ethereum/Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.11-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

vendor/github.com/ethereum/go-ethereum/Makefile

@@ -57,6 +57,9 @@ devtools:
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 
+swarm-devtools:
+	env GOBIN= go install ./cmd/swarm/mimegen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios

vendor/github.com/ethereum/go-ethereum/README.md

@@ -7,7 +7,7 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
 )](https://godoc.org/github.com/ethereum/go-ethereum)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
 
 Automated builds are available for stable releases and the unstable master branch.
 Binary archives are published at https://geth.ethereum.org/downloads/.

vendor/github.com/ethereum/go-ethereum/accounts/abi/abi.go

@@ -137,6 +137,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 // MethodById looks up a method by the 4-byte id
 // returns nil if none found
 func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
+	if len(sigdata) < 4 {
+		return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
+	}
 	for _, method := range abi.Methods {
 		if bytes.Equal(method.Id(), sigdata[:4]) {
 			return &method, nil

vendor/github.com/ethereum/go-ethereum/accounts/abi/abi_test.go

@@ -711,5 +711,14 @@ func TestABI_MethodById(t *testing.T) {
 			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
 		}
 	}
-
+	// Also test empty
+	if _, err := abi.MethodById([]byte{0x00}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById([]byte{}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById(nil); err == nil {
+		t.Errorf("Expected error, nil is short to decode data")
+	}
 }

vendor/github.com/ethereum/go-ethereum/accounts/abi/bind/backends/simulated.go

@@ -69,7 +69,7 @@ func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBac
 	database := ethdb.NewMemDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
@@ -208,7 +208,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad
 }
 
 // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
-// chain doens't have miners, we just return a gas price of 1 for any call.
+// chain doesn't have miners, we just return a gas price of 1 for any call.
 func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	return big.NewInt(1), nil
 }

vendor/github.com/ethereum/go-ethereum/accounts/abi/bind/bind.go

@@ -23,13 +23,13 @@ package bind
 import (
 	"bytes"
 	"fmt"
+	"go/format"
 	"regexp"
 	"strings"
 	"text/template"
 	"unicode"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
-	"golang.org/x/tools/imports"
 )
 
 // Lang is a target programming language selector to generate bindings for.
@@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	if err := tmpl.Execute(buffer, data); err != nil {
 		return "", err
 	}
-	// For Go bindings pass the code through goimports to clean it up and double check
+	// For Go bindings pass the code through gofmt to clean it up
 	if lang == LangGo {
-		code, err := imports.Process(".", buffer.Bytes(), nil)
+		code, err := format.Source(buffer.Bytes())
 		if err != nil {
 			return "", fmt.Errorf("%v\n%s", err, buffer)
 		}

vendor/github.com/ethereum/go-ethereum/accounts/abi/bind/template.go

@@ -64,6 +64,30 @@ const tmplSourceGo = `
 
 package {{.Package}}
 
+import (
+	"math/big"
+	"strings"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var (
+	_ = big.NewInt
+	_ = strings.NewReader
+	_ = ethereum.NotFound
+	_ = abi.U256
+	_ = bind.Bind
+	_ = common.Big1
+	_ = types.BloomLookup
+	_ = event.NewSubscription
+)
+
 {{range $contract := .Contracts}}
 	// {{.Type}}ABI is the input ABI used to generate the binding from.
 	const {{.Type}}ABI = "{{.InputABI}}"

vendor/github.com/ethereum/go-ethereum/accounts/abi/type.go

@@ -103,7 +103,12 @@ func NewType(t string) (typ Type, err error) {
 		return typ, err
 	}
 	// parse the type and size of the abi-type.
-	parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
+	matches := typeRegex.FindAllStringSubmatch(t, -1)
+	if len(matches) == 0 {
+		return Type{}, fmt.Errorf("invalid type '%v'", t)
+	}
+	parsedType := matches[0]
+
 	// varSize is the size of the variable
 	var varSize int
 	if len(parsedType[3]) > 0 {

vendor/github.com/ethereum/go-ethereum/accounts/abi/unpack.go

@@ -25,8 +25,17 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+var (
+	maxUint256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
+		big.NewInt(-1))
+	maxInt256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
+		big.NewInt(-1))
+)
+
 // reads the integer based on its kind
-func readInteger(kind reflect.Kind, b []byte) interface{} {
+func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
 		return b[len(b)-1]
@@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	case reflect.Int64:
 		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
 	default:
-		return new(big.Int).SetBytes(b)
+		// the only case lefts for integer is int256/uint256.
+		// big.SetBytes can't tell if a number is negative, positive on itself.
+		// On EVM, if the returned number > max int256, it is negative.
+		ret := new(big.Int).SetBytes(b)
+		if typ == UintTy {
+			return ret
+		}
+
+		if ret.Cmp(maxInt256) > 0 {
+			ret.Add(maxUint256, big.NewInt(0).Neg(ret))
+			ret.Add(ret, big.NewInt(1))
+			ret.Neg(ret)
+		}
+		return ret
 	}
 }
 
@@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	case StringTy: // variable arrays are written at the end of the return bytes
 		return string(output[begin : begin+end]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Kind, returnOutput), nil
+		return readInteger(t.T, t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:

vendor/github.com/ethereum/go-ethereum/accounts/abi/unpack_test.go

@@ -117,6 +117,11 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: big.NewInt(1),
 	},
+	{
+		def:  `[{"type": "int256"}]`,
+		enc:  "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		want: big.NewInt(-1),
+	},
 	{
 		def:  `[{"type": "address"}]`,
 		enc:  "0000000000000000000000000100000000000000000000000000000000000000",

vendor/github.com/ethereum/go-ethereum/accounts/hd.go

@@ -30,8 +30,8 @@ import (
 var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
-// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
+// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
-// at m/44'/60'/0'/1, etc.
+// at m/44'/60'/0'/0/1, etc.
 var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
 
 // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

vendor/github.com/ethereum/go-ethereum/accounts/keystore/key.go

@@ -66,19 +66,19 @@ type plainKeyJSON struct {
 
 type encryptedKeyJSONV3 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version int        `json:"version"`
 }
 
 type encryptedKeyJSONV1 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version string     `json:"version"`
 }
 
-type cryptoJSON struct {
+type CryptoJSON struct {
 	Cipher       string                 `json:"cipher"`
 	CipherText   string                 `json:"ciphertext"`
 	CipherParams cipherparamsJSON       `json:"cipherparams"`
@@ -179,26 +179,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	return key, a, err
 }
 
-func writeKeyFile(file string, content []byte) error {
+func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	// Create the keystore directory with appropriate permissions
 	// in case it is not present yet.
 	const dirPerm = 0700
 	if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
-		return err
+		return "", err
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
 	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
-		return err
+		return "", err
 	}
 	if _, err := f.Write(content); err != nil {
 		f.Close()
 		os.Remove(f.Name())
-		return err
+		return "", err
 	}
 	f.Close()
-	return os.Rename(f.Name(), file)
+	return f.Name(), nil
+}
+
+func writeKeyFile(file string, content []byte) error {
+	name, err := writeTemporaryKeyFile(file, content)
+	if err != nil {
+		return err
+	}
+	return os.Rename(name, file)
 }
 
 // keyFileName implements the naming convention for keyfiles:

vendor/github.com/ethereum/go-ethereum/accounts/keystore/keystore.go

@@ -78,7 +78,7 @@ type unlocked struct {
 // NewKeyStore creates a keystore for the given directory.
 func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
 	keydir, _ = filepath.Abs(keydir)
-	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
 	ks.init(keydir)
 	return ks
 }

vendor/github.com/ethereum/go-ethereum/accounts/keystore/keystore_passphrase.go

@@ -35,6 +35,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -72,6 +73,10 @@ type keyStorePassphrase struct {
 	keysDirPath string
 	scryptN     int
 	scryptP     int
+	// skipKeyFileVerification disables the security-feature which does
+	// reads and decrypts any newly created keyfiles. This should be 'false' in all
+	// cases except tests -- setting this to 'true' is not recommended.
+	skipKeyFileVerification bool
 }
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
@@ -93,7 +98,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 
 // StoreKey generates a key, encrypts with 'auth' and stores in the given directory
 func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
-	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, rand.Reader, auth)
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
 	return a.Address, err
 }
 
@@ -102,7 +107,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
 	if err != nil {
 		return err
 	}
-	return writeKeyFile(filename, keyjson)
+	// Write into temporary file
+	tmpName, err := writeTemporaryKeyFile(filename, keyjson)
+	if err != nil {
+		return err
+	}
+	if !ks.skipKeyFileVerification {
+		// Verify that we can decrypt the file with the given password.
+		_, err = ks.GetKey(key.Address, tmpName, auth)
+		if err != nil {
+			msg := "An error was encountered when saving and verifying the keystore file. \n" +
+				"This indicates that the keystore is corrupted. \n" +
+				"The corrupted file is stored at \n%v\n" +
+				"Please file a ticket at:\n\n" +
+				"https://github.com/ethereum/go-ethereum/issues." +
+				"The error was : %s"
+			return fmt.Errorf(msg, tmpName, err)
+		}
+	}
+	return os.Rename(tmpName, filename)
 }
 
 func (ks keyStorePassphrase) JoinPath(filename string) string {
@@ -112,29 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 	return filepath.Join(ks.keysDirPath, filename)
 }
 
-// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// Encryptdata encrypts the data given as 'data' with the password 'auth'.
-// blob that can be decrypted later on.
+func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
-func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
-	authArray := []byte(auth)
 
 	salt := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen)
+	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
 	iv := make([]byte, aes.BlockSize) // 16
 	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
+	cipherText, err := aesCTRXOR(encryptKey, data, iv)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
 
@@ -144,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 	scryptParamsJSON["p"] = scryptP
 	scryptParamsJSON["dklen"] = scryptDKLen
 	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
-
 	cipherParamsJSON := cipherparamsJSON{
 		IV: hex.EncodeToString(iv),
 	}
 
-	cryptoStruct := cryptoJSON{
+	cryptoStruct := CryptoJSON{
 		Cipher:       "aes-128-ctr",
 		CipherText:   hex.EncodeToString(cipherText),
 		CipherParams: cipherParamsJSON,
@@ -157,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		KDFParams:    scryptParamsJSON,
 		MAC:          hex.EncodeToString(mac),
 	}
+	return cryptoStruct, nil
+}
+
+// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// blob that can be decrypted later on.
+func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
+	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
 	encryptedKeyJSONV3 := encryptedKeyJSONV3{
 		hex.EncodeToString(key.Address[:]),
 		cryptoStruct,
@@ -203,43 +233,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 		PrivateKey: key,
 	}, nil
 }
+func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
+	if cryptoJson.Cipher != "aes-128-ctr" {
+		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)
+	}
+	mac, err := hex.DecodeString(cryptoJson.MAC)
+	if err != nil {
+		return nil, err
+	}
+
+	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
+	if err != nil {
+		return nil, err
+	}
+
+	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
+	if err != nil {
+		return nil, err
+	}
+
+	derivedKey, err := getKDFKey(cryptoJson, auth)
+	if err != nil {
+		return nil, err
+	}
+
+	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
+	if !bytes.Equal(calculatedMAC, mac) {
+		return nil, ErrDecrypt
+	}
+
+	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
+	if err != nil {
+		return nil, err
+	}
+	return plainText, err
+}
 
 func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
 	if keyProtected.Version != version {
 		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
 	}
-
-	if keyProtected.Crypto.Cipher != "aes-128-ctr" {
-		return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
-	}
-
 	keyId = uuid.Parse(keyProtected.Id)
-	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
+	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
-	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, nil, ErrDecrypt
-	}
-
-	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -280,7 +315,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt
 	return plainText, keyId, err
 }
 
-func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
+func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
 	authArray := []byte(auth)
 	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
 	if err != nil {

vendor/github.com/ethereum/go-ethereum/accounts/keystore/keystore_plain_test.go

@@ -37,7 +37,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 		t.Fatal(err)
 	}
 	if encrypted {
-		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP}
+		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
 	} else {
 		ks = &keyStorePlain{d}
 	}
@@ -191,7 +191,7 @@ func TestV1_1(t *testing.T) {
 
 func TestV1_2(t *testing.T) {
 	t.Parallel()
-	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP}
+	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
 	addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
 	file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
 	k, err := ks.GetKey(addr, file, "g")

vendor/github.com/ethereum/go-ethereum/accounts/usbwallet/ledger.go

@@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

vendor/github.com/ethereum/go-ethereum/accounts/usbwallet/trezor.go

@@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

vendor/github.com/ethereum/go-ethereum/appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.3.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
-  - 7z x go1.10.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

vendor/github.com/ethereum/go-ethereum/build/ci.go

@@ -320,9 +320,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
 // "tests" also includes static analysis tools such as vet.
 
 func doTest(cmdline []string) {
-	var (
+	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
-		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
-	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -332,14 +330,11 @@ func doTest(cmdline []string) {
 	}
 	packages = build.ExpandPackagesNoVendor(packages)
 
-	// Run analysis tools before the tests.
-	build.MustRun(goTool("vet", packages...))
-
 	// Run the actual tests.
-	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest.Args = append(gotest.Args, "-p", "1")
+	gotest := goTool("test", buildFlags(env)...)
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -1040,7 +1035,7 @@ func xgoTool(args []string) *exec.Cmd {
 func doPurge(cmdline []string) {
 	var (
 		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
-		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
 	)
 	flag.CommandLine.Parse(cmdline)
 

vendor/github.com/ethereum/go-ethereum/cmd/abigen/main.go

@@ -75,7 +75,7 @@ func main() {
 		bins  []string
 		types []string
 	)
-	if *solFlag != "" || *abiFlag == "-" {
+	if *solFlag != "" || (*abiFlag == "-" && *pkgFlag == "") {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
 		for _, kind := range strings.Split(*excFlag, ",") {
@@ -111,7 +111,13 @@ func main() {
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters
-		abi, err := ioutil.ReadFile(*abiFlag)
+		var abi []byte
+		var err error
+		if *abiFlag == "-" {
+			abi, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			abi, err = ioutil.ReadFile(*abiFlag)
+		}
 		if err != nil {
 			fmt.Printf("Failed to read input ABI: %v\n", err)
 			os.Exit(-1)
@@ -155,6 +161,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
 }

vendor/github.com/ethereum/go-ethereum/cmd/bootnode/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
@@ -37,7 +38,7 @@ func main() {
 	var (
 		listenAddr  = flag.String("addr", ":30301", "listen address")
 		genKey      = flag.String("genkey", "", "generate a node key")
-		writeAddr   = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit")
+		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -85,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", discover.PubkeyID(&nodeKey.PublicKey))
+		fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
 		os.Exit(0)
 	}
 
@@ -118,16 +119,17 @@ func main() {
 	}
 
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
+		db, _ := enode.OpenDB("")
+		ln := enode.NewLocalNode(db, nodeKey)
 		cfg := discover.Config{
 			PrivateKey:  nodeKey,
-			AnnounceAddr: realaddr,
 			NetRestrict: restrictList,
 		}
-		if _, err := discover.ListenUDP(conn, cfg); err != nil {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

vendor/github.com/ethereum/go-ethereum/cmd/clef/README.md

@@ -91,7 +91,7 @@ invoking methods with the following info:
   * [x] Version info about the signer
   * [x] Address of API (http/ipc)
   * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
 * [ ] `account_bulkSignTransactions([] transactions)` should
    * only exist if enabled via config/flag
@@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
 expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
 Some of these call can require user interaction. Clients must be aware that responses
-may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
 The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
 that requests have any authority.
@@ -862,7 +862,7 @@ A UI should conform to the following rules.
 * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
 * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
   * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
 along with the UI.
 
 
@@ -875,3 +875,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
 | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
 | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
+| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

vendor/github.com/ethereum/go-ethereum/cmd/clef/extapi_changelog.md

@@ -1,6 +1,13 @@
 ### Changelog for external API
 
+#### 4.0.0
 
+* The external `account_Ecrecover`-method was removed. 
+* The external `account_Import`-method was removed.
+
+#### 3.0.0
+
+* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses. 
 
 #### 2.0.0
 

vendor/github.com/ethereum/go-ethereum/cmd/clef/intapi_changelog.md

@@ -1,5 +1,24 @@
 ### Changelog for internal API (ui-api)
 
+### 3.0.0
+
+* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
+
+### 2.1.0
+
+* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
+
+The following structures are used:
+```golang
+       UserInputRequest struct {
+               Prompt     string `json:"prompt"`
+               Title      string `json:"title"`
+               IsPassword bool   `json:"isPassword"`
+       }
+       UserInputResponse struct {
+               Text string `json:"text"`
+       }
+
 ### 2.0.0
 
 * Modify how `call_info` on a transaction is conveyed. New format:

vendor/github.com/ethereum/go-ethereum/cmd/clef/main.go

@@ -35,8 +35,10 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
@@ -48,10 +50,10 @@ import (
 )
 
 // ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "2.0.0"
+const ExternalAPIVersion = "4.0.0"
 
 // InternalAPIVersion -- see intapi_changelog.md
-const InternalAPIVersion = "2.0.0"
+const InternalAPIVersion = "3.0.0"
 
 const legalWarning = `
 WARNING! 
@@ -70,6 +72,10 @@ var (
 		Value: 4,
 		Usage: "log level to emit to the screen",
 	}
+	advancedMode = cli.BoolFlag{
+		Name:  "advanced",
+		Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
+	}
 	keystoreFlag = cli.StringFlag{
 		Name:  "keystore",
 		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
@@ -87,7 +93,7 @@ var (
 	}
 	signerSecretFlag = cli.StringFlag{
 		Name:  "signersecret",
-		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
 	}
 	dBFlag = cli.StringFlag{
 		Name:  "4bytedb",
@@ -151,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
 
-	addCredentialCommand = cli.Command{
+	setCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(addCredential),
+		Action:    utils.MigrateFlags(setCredential),
-		Name:      "addpw",
+		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
-		ArgsUsage: "<address> <password>",
+		ArgsUsage: "<address>",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			configdirFlag,
 			signerSecretFlag,
 		},
 		Description: `
-The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+		The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
 `,
 	}
@@ -191,9 +197,10 @@ func init() {
 		ruleFlag,
 		stdiouiFlag,
 		testFlag,
+		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
 
 }
 func main() {
@@ -207,25 +214,45 @@ func initializeSecrets(c *cli.Context) error {
 	if err := initialize(c); err != nil {
 		return err
 	}
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
 
 	masterSeed := make([]byte, 256)
-	n, err := io.ReadFull(rand.Reader, masterSeed)
+	num, err := io.ReadFull(rand.Reader, masterSeed)
 	if err != nil {
 		return err
 	}
-	if n != len(masterSeed) {
+	if num != len(masterSeed) {
 		return fmt.Errorf("failed to read enough random")
 	}
+
+	n, p := keystore.StandardScryptN, keystore.StandardScryptP
+	if c.GlobalBool(utils.LightKDFFlag.Name) {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
+	var password string
+	for {
+		password = getPassPhrase(text, true)
+		if err := core.ValidatePasswordFormat(password); err != nil {
+			fmt.Printf("invalid password: %v\n", err)
+		} else {
+			break
+		}
+	}
+	cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt master seed: %v", err)
+	}
+
 	err = os.Mkdir(configDir, 0700)
 	if err != nil && !os.IsExist(err) {
 		return err
 	}
-	location := filepath.Join(configDir, "secrets.dat")
+	location := filepath.Join(configDir, "masterseed.json")
 	if _, err := os.Stat(location); err == nil {
 		return fmt.Errorf("file %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, masterSeed, 0700)
+	err = ioutil.WriteFile(location, cipherSeed, 0400)
 	if err != nil {
 		return err
 	}
@@ -250,11 +277,11 @@ func attestFile(ctx *cli.Context) error {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
 
@@ -266,38 +293,36 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
-func addCredential(ctx *cli.Context) error {
+func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires at leaste one argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument.")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	address := ctx.Args().First()
+	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
 	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	key := ctx.Args().First()
+	pwStorage.Put(address, password)
-	value := ""
+	log.Info("Credential store updated", "key", address)
-	if len(ctx.Args()) > 1 {
-		value = ctx.Args().Get(1)
-	}
-	pwStorage.Put(key, value)
-	log.Info("Credential store updated", "key", key)
 	return nil
 }
 
 func initialize(c *cli.Context) error {
 	// Set up the logger to print everything
 	logOutput := os.Stdout
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		logOutput = os.Stderr
 		// If using the stdioui, we can't do the 'confirm'-flow
 		fmt.Fprintf(logOutput, legalWarning)
@@ -318,26 +343,28 @@ func signer(c *cli.Context) error {
 	var (
 		ui core.SignerUI
 	)
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
 		ui = core.NewStdIOUI()
 	} else {
 		log.Info("Using CLI as UI-channel")
 		ui = core.NewCommandlineUI()
 	}
-	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	fourByteDb := c.GlobalString(dBFlag.Name)
+	fourByteLocal := c.GlobalString(customDBFlag.Name)
+	db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal)
 
 	var (
 		api core.ExternalAPI
 	)
 
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
-	if stretchedKey, err := readMasterKey(c); err != nil {
+	if stretchedKey, err := readMasterKey(c, ui); err != nil {
-		log.Info("No master seed provided, rules disabled")
+		log.Info("No master seed provided, rules disabled", "error", err)
 	} else {
 
 		if err != nil {
@@ -356,7 +383,7 @@ func signer(c *cli.Context) error {
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		//Do we have a rule-file?
-		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name))
 		if err != nil {
 			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
 		} else {
@@ -380,16 +407,15 @@ func signer(c *cli.Context) error {
 	}
 
 	apiImpl := core.NewSignerAPI(
-		c.Int64(utils.NetworkIdFlag.Name),
+		c.GlobalInt64(utils.NetworkIdFlag.Name),
-		c.String(keystoreFlag.Name),
+		c.GlobalString(keystoreFlag.Name),
-		c.Bool(utils.NoUSBFlag.Name),
+		c.GlobalBool(utils.NoUSBFlag.Name),
 		ui, db,
-		c.Bool(utils.LightKDFFlag.Name))
+		c.GlobalBool(utils.LightKDFFlag.Name),
-
+		c.GlobalBool(advancedMode.Name))
 	api = apiImpl
-
 	// Audit logging
-	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
 		api, err = core.NewAuditLogger(logfile, api)
 		if err != nil {
 			utils.Fatalf(err.Error())
@@ -408,13 +434,13 @@ func signer(c *cli.Context) error {
 			Service:   api,
 			Version:   "1.0"},
 	}
-	if c.Bool(utils.RPCEnabledFlag.Name) {
+	if c.GlobalBool(utils.RPCEnabledFlag.Name) {
 
 		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
 		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
 
 		// start http server
-		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
 		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
 		if err != nil {
 			utils.Fatalf("Could not start RPC api: %v", err)
@@ -428,9 +454,9 @@ func signer(c *cli.Context) error {
 		}()
 
 	}
-	if !c.Bool(utils.IPCDisabledFlag.Name) {
+	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
 		if c.IsSet(utils.IPCPathFlag.Name) {
-			ipcapiURL = c.String(utils.IPCPathFlag.Name)
+			ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name)
 		} else {
 			ipcapiURL = filepath.Join(configDir, "clef.ipc")
 		}
@@ -447,7 +473,7 @@ func signer(c *cli.Context) error {
 
 	}
 
-	if c.Bool(testFlag.Name) {
+	if c.GlobalBool(testFlag.Name) {
 		log.Info("Performing UI test")
 		go testExternalUI(apiImpl)
 	}
@@ -506,48 +532,64 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
 	var (
 		file      string
-		configDir = ctx.String(configdirFlag.Name)
+		configDir = ctx.GlobalString(configdirFlag.Name)
 	)
-	if ctx.IsSet(signerSecretFlag.Name) {
+	if ctx.GlobalIsSet(signerSecretFlag.Name) {
-		file = ctx.String(signerSecretFlag.Name)
+		file = ctx.GlobalString(signerSecretFlag.Name)
 	} else {
-		file = filepath.Join(configDir, "secrets.dat")
+		file = filepath.Join(configDir, "masterseed.json")
 	}
 	if err := checkFile(file); err != nil {
 		return nil, err
 	}
-	masterKey, err := ioutil.ReadFile(file)
+	cipherKey, err := ioutil.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
-	if len(masterKey) < 256 {
+	var password string
-		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	// If ui is not nil, get the password from ui.
+	if ui != nil {
+		resp, err := ui.OnInputRequired(core.UserInputRequest{
+			Title:      "Master Password",
+			Prompt:     "Please enter the password to decrypt the master seed",
+			IsPassword: true})
+		if err != nil {
+			return nil, err
 		}
+		password = resp.Text
+	} else {
+		password = getPassPhrase("Decrypt master seed of clef", false)
+	}
+	masterSeed, err := decryptSeed(cipherKey, password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt the master seed of clef")
+	}
+	if len(masterSeed) < 256 {
+		return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
+	}
+
 	// Create vault location
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
 	err = os.Mkdir(vaultLocation, 0700)
 	if err != nil && !os.IsExist(err) {
 		return nil, err
 	}
-	//!TODO, use KDF to stretch the master key
+	return masterSeed, nil
-	//			stretched_key := stretch_key(master_key)
-
-	return masterKey, nil
 }
 
 // checkFile is a convenience function to check if a file
 // * exists
-// * is mode 0600
+// * is mode 0400
 func checkFile(filename string) error {
 	info, err := os.Stat(filename)
 	if err != nil {
 		return fmt.Errorf("failed stat on %s: %v", filename, err)
 	}
 	// Check the unix permission bits
-	if info.Mode().Perm()&077 != 0 {
+	if info.Mode().Perm()&0377 != 0 {
 		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
 	}
 	return nil
@@ -613,6 +655,59 @@ func testExternalUI(api *core.SignerAPI) {
 
 }
 
+// getPassPhrase retrieves the password associated with clef, either fetched
+// from a list of preloaded passphrases, or requested interactively from the user.
+// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
+func getPassPhrase(prompt string, confirmation bool) string {
+	fmt.Println(prompt)
+	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	if err != nil {
+		utils.Fatalf("Failed to read passphrase: %v", err)
+	}
+	if confirmation {
+		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+		}
+		if password != confirm {
+			utils.Fatalf("Passphrases do not match")
+		}
+	}
+	return password
+}
+
+type encryptedSeedStorage struct {
+	Description string              `json:"description"`
+	Version     int                 `json:"version"`
+	Params      keystore.CryptoJSON `json:"params"`
+}
+
+// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping,
+// to encrypt the master seed
+func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) {
+	cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct})
+}
+
+// decryptSeed decrypts the master seed
+func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
+	var encSeed encryptedSeedStorage
+	if err := json.Unmarshal(keyjson, &encSeed); err != nil {
+		return nil, err
+	}
+	if encSeed.Version != 1 {
+		log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version))
+	}
+	seed, err := keystore.DecryptDataV3(encSeed.Params, auth)
+	if err != nil {
+		return nil, err
+	}
+	return seed, err
+}
+
 /**
 //Create Account
 

vendor/github.com/ethereum/go-ethereum/cmd/clef/tutorial.md

@@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ
 
 ## Creating rules
 
-Now, you can create a rule-file.
+Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
 
 ```javascript
 function ApproveListing(){
     return "Approve"
 }
 ```
-Get the `sha256` hash....
+
+Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
 ```text
 #sha256sum rules.js
 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
 ```
-...And then `attest` the file:
+...now `attest` the file...
 ```text
 #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 
 INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 ```
-At this point, we then start the signer with the rule-file:
 
+...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
 ```text
-#./signer --rules rules.json
+#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
+```
 
-INFO [02-21|12:15:18] Using CLI as UI-channel
+At this point, we can start the signer with the rule-file:
-INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
+```text
-INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
+#./signer --rules rules.js --rpc
-DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
+
-DEBUG[02-21|12:15:18] Ledger support enabled
+INFO [09-25|20:28:11.866] Using CLI as UI-channel 
-DEBUG[02-21|12:15:18] Trezor support enabled
+INFO [09-25|20:28:11.876] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
+INFO [09-25|20:28:11.877] Rule engine configured                   file=./rules.js
-INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.877] FS scan times                            list=100.781µs set=13.253µs diff=5.761µs
+DEBUG[09-25|20:28:11.884] Ledger support enabled 
+DEBUG[09-25|20:28:11.888] Trezor support enabled 
+INFO [09-25|20:28:11.888] Audit logs configured                    file=audit.log
+DEBUG[09-25|20:28:11.888] HTTP registered                          namespace=account
+INFO [09-25|20:28:11.890] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.890] IPC registered                           namespace=account
+INFO [09-25|20:28:11.890] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-* extapi_version : 2.0.0
-* intapi_version : 1.2.0
-
 ```
 
 Any list-requests will now be auto-approved by our rule-file.
@@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things:
 
 ## Adding credentials
 
-In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
 
 ```text
-#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
 
 INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
 ```
 ## More advanced rules
 
-Now let's update the rules to make use of credentials
+Now let's update the rules to make use of credentials:
 
 ```javascript
 function ApproveListing(){
@@ -134,13 +142,15 @@ function ApproveSignData(r){
 }
 
 ```
-In this example,
+In this example:
-* any requests to sign data with the account `0x694...` will be
+* Any requests to sign data with the account `0x694...` will be
-    * auto-approved if the message contains with `bazonk`,
+    * auto-approved if the message contains with `bazonk`
-    * and auto-rejected if it does not.
+    * auto-rejected if it does not.
 * Any other signing-requests will be passed along for manual approve/reject.
 
-..attest the new file
+_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
+
+Attest the new file...
 ```text
 #sha256sum rules.js
 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
@@ -153,23 +163,26 @@ INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661da
 And start the signer:
 
 ```
-#./signer --rules rules.js
+#./signer --rules rules.js --rpc
 
-INFO [02-21|14:41:56] Using CLI as UI-channel
+INFO [09-25|21:02:16.450] Using CLI as UI-channel 
-INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|21:02:16.466] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
+INFO [09-25|21:02:16.467] Rule engine configured                   file=./rules.js
-DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
+DEBUG[09-25|21:02:16.468] FS scan times                            list=1.45262ms set=21.926µs diff=6.944µs
-DEBUG[02-21|14:41:56] Ledger support enabled
+DEBUG[09-25|21:02:16.473] Ledger support enabled 
-DEBUG[02-21|14:41:56] Trezor support enabled
+DEBUG[09-25|21:02:16.475] Trezor support enabled 
-INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
+INFO [09-25|21:02:16.476] Audit logs configured                    file=audit.log
-INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.476] HTTP registered                          namespace=account
+INFO [09-25|21:02:16.478] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.478] IPC registered                           namespace=account
+INFO [09-25|21:02:16.478] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
 * extapi_version : 2.0.0
-* intapi_version : 1.2.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
 ```
+
 And then test signing, once with `bazonk` and once without:
 
 ```
@@ -190,7 +203,7 @@ INFO [02-21|14:42:56] Op rejected
 The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
 
 ```text
-#tail audit.log -n 4
+#tail -n 4 audit.log
 t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a

vendor/github.com/ethereum/go-ethereum/cmd/ethkey/README.md

@@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag;
 make sure to use this feature with great caution!
 
 
-### `ethkey sign <keyfile> <message/file>`
+### `ethkey signmessage <keyfile> <message/file>`
 
 Sign the message with a keyfile.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the `--msgfile` flag.
 
 
-### `ethkey verify <address> <signature> <message/file>`
+### `ethkey verifymessage <address> <signature> <message/file>`
 
 Verify the signature of the message.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the --msgfile flag.
+
+
+### `ethkey changepassphrase <keyfile>`
+
+Change the passphrase of a keyfile.
+use the `--newpasswordfile` to point to the new password file.
 
 
 ## Passphrases
 
 For every command that uses a keyfile, you will be prompted to provide the 
 passphrase for decrypting the keyfile.  To avoid this message, it is possible
-to pass the passphrase by using the `--passphrase` flag pointing to a file that
+to pass the passphrase by using the `--passwordfile` flag pointing to a file that
 contains the passphrase.
+
+## JSON
+
+In case you need to output the result in a JSON format, you shall by using the `--json` flag.

vendor/github.com/ethereum/go-ethereum/cmd/evm/disasm.go

@@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error {
 		return err
 	}
 
-	code := strings.TrimSpace(string(in[:]))
+	code := strings.TrimSpace(string(in))
 	fmt.Printf("%v\n", code)
 	return asm.PrintDisassembled(code)
 }

vendor/github.com/ethereum/go-ethereum/cmd/evm/json_logger.go

@@ -52,6 +52,7 @@ func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cos
 		MemorySize:    memory.Len(),
 		Storage:       nil,
 		Depth:         depth,
+		RefundCounter: env.StateDB.GetRefund(),
 		Err:           err,
 	}
 	if !l.cfg.DisableMemory {

vendor/github.com/ethereum/go-ethereum/cmd/evm/runner.go

@@ -86,7 +86,7 @@ func runCmd(ctx *cli.Context) error {
 		chainConfig   *params.ChainConfig
 		sender        = common.BytesToAddress([]byte("sender"))
 		receiver      = common.BytesToAddress([]byte("receiver"))
-		blockNumber uint64
+		genesisConfig *core.Genesis
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
 		tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -98,13 +98,14 @@ func runCmd(ctx *cli.Context) error {
 	}
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
+		genesisConfig = gen
 		db := ethdb.NewMemDatabase()
 		genesis := gen.ToBlock(db)
 		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
 		chainConfig = gen.Config
-		blockNumber = gen.Number
 	} else {
 		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		genesisConfig = new(core.Genesis)
 	}
 	if ctx.GlobalString(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
@@ -156,13 +157,19 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	initialGas := ctx.GlobalUint64(GasFlag.Name)
+	if genesisConfig.GasLimit != 0 {
+		initialGas = genesisConfig.GasLimit
+	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
 		GasLimit:    initialGas,
 		GasPrice:    utils.GlobalBig(ctx, PriceFlag.Name),
 		Value:       utils.GlobalBig(ctx, ValueFlag.Name),
-		BlockNumber: new(big.Int).SetUint64(blockNumber),
+		Difficulty:  genesisConfig.Difficulty,
+		Time:        new(big.Int).SetUint64(genesisConfig.Timestamp),
+		Coinbase:    genesisConfig.Coinbase,
+		BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
 		EVMConfig: vm.Config{
 			Tracer: tracer,
 			Debug:  ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),

vendor/github.com/ethereum/go-ethereum/cmd/evm/staterunner.go

@@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
 			state, err := test.Run(st, cfg)
+			// print state root for evmlab tracing
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
 			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
@@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
-			// print state root for evmlab tracing (already committed above, so no need to delete objects again
-			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
-				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
-			}
 
 			results = append(results, *result)
 

vendor/github.com/ethereum/go-ethereum/cmd/faucet/faucet.go

@@ -54,8 +54,8 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
 	"golang.org/x/net/websocket"
@@ -157,7 +157,8 @@ func main() {
 	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
 		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
 	}
-	pass := string(blob)
+	// Delete trailing newline in password
+	pass := strings.TrimSuffix(string(blob), "\n")
 
 	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
 	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
@@ -198,6 +199,8 @@ type faucet struct {
 
 	keystore *keystore.KeyStore // Keystore containing the single signer
 	account  accounts.Account   // Account funding user faucet requests
+	head     *types.Header      // Current head header of the faucet
+	balance  *big.Int           // Current balance of the faucet
 	nonce    uint64             // Current pending nonce of the faucet
 	price    *big.Int           // Current gas price to issue funds with
 
@@ -252,9 +255,11 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 		return nil, err
 	}
 	for _, boot := range enodes {
-		old, _ := discover.ParseNode(boot.String())
+		old, err := enode.ParseV4(boot.String())
+		if err != nil {
 			stack.Server().AddPeer(old)
 		}
+	}
 	// Attach to the client and retrieve and interesting metadatas
 	api, err := stack.Attach()
 	if err != nil {
@@ -323,33 +328,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 		nonce   uint64
 		err     error
 	)
-	for {
+	for head == nil || balance == nil {
-		// Attempt to retrieve the stats, may error on no faucet connectivity
+		// Retrieve the current stats cached by the faucet
-		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		f.lock.RLock()
-		head, err = f.client.HeaderByNumber(ctx, nil)
+		if f.head != nil {
-		if err == nil {
+			head = types.CopyHeader(f.head)
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
 		}
+		if f.balance != nil {
+			balance = new(big.Int).Set(f.balance)
 		}
-		cancel()
+		nonce = f.nonce
+		f.lock.RUnlock()
 
-		// If stats retrieval failed, wait a bit and retry
+		if head == nil || balance == nil {
-		if err != nil {
+			// Report the faucet offline until initial stats are ready
-			if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
+			if err = sendError(conn, errors.New("Faucet offline")); err != nil {
 				log.Warn("Failed to send faucet error to client", "err", err)
 				return
 			}
 			time.Sleep(3 * time.Second)
-			continue
 		}
-		// Initial stats reported successfully, proceed with user interaction
-		break
 	}
 	// Send over the initial stats and the latest header
 	if err = send(conn, map[string]interface{}{
-		"funds":    balance.Div(balance, ether),
+		"funds":    new(big.Int).Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
 		"requests": f.reqs,
@@ -519,6 +521,47 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 	}
 }
 
+// refresh attempts to retrieve the latest header from the chain and extract the
+// associated faucet balance and nonce for connectivity caching.
+func (f *faucet) refresh(head *types.Header) error {
+	// Ensure a state update does not run for too long
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// If no header was specified, use the current chain head
+	var err error
+	if head == nil {
+		if head, err = f.client.HeaderByNumber(ctx, nil); err != nil {
+			return err
+		}
+	}
+	// Retrieve the balance, nonce and gas price from the current head
+	var (
+		balance *big.Int
+		nonce   uint64
+		price   *big.Int
+	)
+	if balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if nonce, err = f.client.NonceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if price, err = f.client.SuggestGasPrice(ctx); err != nil {
+		return err
+	}
+	// Everything succeeded, update the cached stats and eject old requests
+	f.lock.Lock()
+	f.head, f.balance = head, balance
+	f.price, f.nonce = price, nonce
+	for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+		f.reqs = f.reqs[1:]
+	}
+	f.lock.Unlock()
+
+	return nil
+}
+
 // loop keeps waiting for interesting events and pushes them out to connected
 // websockets.
 func (f *faucet) loop() {
@@ -536,45 +579,27 @@ func (f *faucet) loop() {
 	go func() {
 		for head := range update {
 			// New chain head arrived, query the current stats and stream to clients
-			var (
+			timestamp := time.Unix(head.Time.Int64(), 0)
-				balance *big.Int
+			if time.Since(timestamp) > time.Hour {
-				nonce   uint64
+				log.Warn("Skipping faucet refresh, head too old", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp))
-				price   *big.Int
+				continue
-				err     error
-			)
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-				if err == nil {
-					price, err = f.client.SuggestGasPrice(ctx)
 			}
-			}
+			if err := f.refresh(head); err != nil {
-			cancel()
-
-			// If querying the data failed, try for the next block
-			if err != nil {
 				log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
 				continue
-			} else {
-				log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
 			}
 			// Faucet state retrieved, update locally and send to clients
-			balance = new(big.Int).Div(balance, ether)
-
-			f.lock.Lock()
-			f.price, f.nonce = price, nonce
-			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
-				f.reqs = f.reqs[1:]
-			}
-			f.lock.Unlock()
-
 			f.lock.RLock()
+			log.Info("Updated faucet state", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp), "balance", f.balance, "nonce", f.nonce, "price", f.price)
+
+			balance := new(big.Int).Div(f.balance, ether)
+			peers := f.stack.Server().PeerCount()
+
 			for _, conn := range f.conns {
 				if err := send(conn, map[string]interface{}{
 					"funds":    balance,
 					"funded":   f.nonce,
-					"peers":    f.stack.Server().PeerCount(),
+					"peers":    peers,
 					"requests": f.reqs,
 				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)

vendor/github.com/ethereum/go-ethereum/cmd/geth/chaincmd.go

@@ -340,9 +340,9 @@ func importPreimages(ctx *cli.Context) error {
 
 	start := time.Now()
 	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
-		utils.Fatalf("Export error: %v\n", err)
+		utils.Fatalf("Import error: %v\n", err)
 	}
-	fmt.Printf("Export done in %v\n", time.Since(start))
+	fmt.Printf("Import done in %v\n", time.Since(start))
 	return nil
 }
 

vendor/github.com/ethereum/go-ethereum/cmd/geth/config.go

@@ -168,6 +168,9 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) {
 			cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name)
 		}
+		if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
+			cfg.Shh.RestrictConnectionBetweenLightClients = true
+		}
 		utils.RegisterShhService(stack, &cfg.Shh)
 	}
 

vendor/github.com/ethereum/go-ethereum/cmd/geth/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -130,6 +129,8 @@ var (
 		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
 		utils.GpoPercentileFlag,
+		utils.EWASMInterpreterFlag,
+		utils.EVMInterpreterFlag,
 		configFileFlag,
 	}
 
@@ -151,6 +152,7 @@ var (
 		utils.WhisperEnabledFlag,
 		utils.WhisperMaxMessageSizeFlag,
 		utils.WhisperMinPOWFlag,
+		utils.WhisperRestrictConnectionBetweenLightClientsFlag,
 	}
 
 	metricsFlags = []cli.Flag{
@@ -206,8 +208,6 @@ func init() {
 	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-
 		logdir := ""
 		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
 			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")

vendor/github.com/ethereum/go-ethereum/cmd/geth/usage.go

@@ -207,6 +207,8 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
 			utils.VMEnableDebugFlag,
+			utils.EVMInterpreterFlag,
+			utils.EWASMInterpreterFlag,
 		},
 	},
 	{

vendor/github.com/ethereum/go-ethereum/cmd/p2psim/main.go

@@ -47,7 +47,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -285,7 +285,7 @@ func createNode(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		config.ID = discover.PubkeyID(&privKey.PublicKey)
+		config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
 		config.PrivateKey = privKey
 	}
 	if services := ctx.String("services"); services != "" {

vendor/github.com/ethereum/go-ethereum/cmd/puppeth/module_node.go

@@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
+	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -99,6 +99,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
 		"NetworkID": config.network,
 		"Port":      config.port,
+		"IP":        client.address,
 		"Peers":     config.peersTotal,
 		"LightFlag": lightFlag,
 		"Bootnodes": strings.Join(bootnodes, ","),
@@ -227,10 +228,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 
 	// Container available, retrieve its node ID and its genesis json
 	var out []byte
-	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil {
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
 	}
-	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+	enode := bytes.Trim(bytes.TrimSpace(out), "\"")
 
 	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
@@ -265,7 +266,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		gasLimit:   gasLimit,
 		gasPrice:   gasPrice,
 	}
-	stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
+	stats.enode = string(enode)
 
 	return stats, nil
 }

vendor/github.com/ethereum/go-ethereum/cmd/puppeth/wizard_dashboard.go

@@ -92,7 +92,7 @@ func (w *wizard) deployDashboard() {
 				pages = append(pages, page)
 			}
 		}
-		// Promt the user to chose one, enter manually or simply not list this service
+		// Prompt the user to chose one, enter manually or simply not list this service
 		defLabel, defChoice := "don't list", len(pages)+2
 		if len(pages) > 0 {
 			defLabel, defChoice = pages[0], 1

vendor/github.com/ethereum/go-ethereum/cmd/puppeth/wizard_network.go

@@ -87,7 +87,7 @@ func (w *wizard) makeServer() string {
 	return input
 }
 
-// selectServer lists the user all the currnetly known servers to choose from,
+// selectServer lists the user all the currently known servers to choose from,
 // also granting the option to add a new one.
 func (w *wizard) selectServer() string {
 	// List the available server to the user and wait for a choice
@@ -115,7 +115,7 @@ func (w *wizard) selectServer() string {
 // manageComponents displays a list of network components the user can tear down
 // and an option
 func (w *wizard) manageComponents() {
-	// List all the componens we can tear down, along with an entry to deploy a new one
+	// List all the components we can tear down, along with an entry to deploy a new one
 	fmt.Println()
 
 	var serviceHosts, serviceNames []string

vendor/github.com/ethereum/go-ethereum/cmd/swarm/access.go

@@ -29,7 +29,65 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-var salt = make([]byte, 32)
+var (
+	salt          = make([]byte, 32)
+	accessCommand = cli.Command{
+		CustomHelpTemplate: helpTemplate,
+		Name:               "access",
+		Usage:              "encrypts a reference and embeds it into a root manifest",
+		ArgsUsage:          "<ref>",
+		Description:        "encrypts a reference and embeds it into a root manifest",
+		Subcommands: []cli.Command{
+			{
+				CustomHelpTemplate: helpTemplate,
+				Name:               "new",
+				Usage:              "encrypts a reference and embeds it into a root manifest",
+				ArgsUsage:          "<ref>",
+				Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+				Subcommands: []cli.Command{
+					{
+						Action:             accessNewPass,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+						},
+						Name:        "pass",
+						Usage:       "encrypts a reference with a password and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewPK,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+							SwarmAccessGrantKeyFlag,
+						},
+						Name:        "pk",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewACT,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							SwarmAccessGrantKeysFlag,
+							SwarmDryRunFlag,
+							utils.PasswordFileFlag,
+						},
+						Name:        "act",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+				},
+			},
+		},
+	}
+)
 
 func init() {
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
@@ -51,18 +109,20 @@ func accessNewPass(ctx *cli.Context) {
 		password  = getPassPhrase("", 0, makePasswordList(ctx))
 		dryRun    = ctx.Bool(SwarmDryRunFlag.Name)
 	)
-	accessKey, ae, err = api.DoPasswordNew(ctx, password, salt)
+	accessKey, ae, err = api.DoPassword(ctx, password, salt)
 	if err != nil {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {
 			utils.Fatalf("had an error printing the manifests: %v", err)
 		}
 	} else {
-		utils.Fatalf("uploading manifests")
 		err = uploadManifests(ctx, m, nil)
 		if err != nil {
 			utils.Fatalf("had an error uploading the manifests: %v", err)
@@ -85,11 +145,14 @@ func accessNewPK(ctx *cli.Context) {
 		granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
 		dryRun           = ctx.Bool(SwarmDryRunFlag.Name)
 	)
-	sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt)
+	sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt)
 	if err != nil {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {
@@ -115,18 +178,33 @@ func accessNewACT(ctx *cli.Context) {
 		accessKey            []byte
 		err                  error
 		ref                  = args[0]
-		grantees    = []string{}
+		pkGrantees           = []string{}
-		actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
+		passGrantees         = []string{}
+		pkGranteesFilename   = ctx.String(SwarmAccessGrantKeysFlag.Name)
+		passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
 		privateKey           = getPrivKey(ctx)
 		dryRun               = ctx.Bool(SwarmDryRunFlag.Name)
 	)
+	if pkGranteesFilename == "" && passGranteesFilename == "" {
+		utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)")
+	}
 
-	bytes, err := ioutil.ReadFile(actFilename)
+	if pkGranteesFilename != "" {
+		bytes, err := ioutil.ReadFile(pkGranteesFilename)
 		if err != nil {
 			utils.Fatalf("had an error reading the grantee public key list")
 		}
-	grantees = strings.Split(string(bytes), "\n")
+		pkGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
-	accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees)
+	}
+
+	if passGranteesFilename != "" {
+		bytes, err := ioutil.ReadFile(passGranteesFilename)
+		if err != nil {
+			utils.Fatalf("could not read password filename: %v", err)
+		}
+		passGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
+	}
+	accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees)
 	if err != nil {
 		utils.Fatalf("error generating ACT manifest: %v", err)
 	}

vendor/github.com/ethereum/go-ethereum/cmd/swarm/access_test.go

@@ -28,50 +28,43 @@ import (
 	gorand "math/rand"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/ecies"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
+const (
+	hashRegexp = `[a-f\d]{128}`
+	data       = "notsorandomdata"
+)
+
+var DefaultCurve = crypto.S256()
+
 // TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
 // is then fetched through 2nd node. since the tested code is not key-aware - we can just
 // fetch from the 2nd node using HTTP BasicAuth
 func TestAccessPassword(t *testing.T) {
-	cluster := newTestCluster(t, 1)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
-	defer cluster.Shutdown()
+	defer srv.Close()
-	proxyNode := cluster.Nodes[0]
 
-	// create a tmp file
+	dataFilename := testutil.TempFileWithContent(t, data)
-	tmp, err := ioutil.TempDir("", "swarm-test")
+	defer os.RemoveAll(dataFilename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-
-	// write data to file
-	data := "notsorandomdata"
-	dataFilename := filepath.Join(tmp, "data.txt")
-
-	err = ioutil.WriteFile(dataFilename, []byte(data), 0666)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hashRegexp := `[a-f\d]{128}`
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
 		"--bzzapi",
-		proxyNode.URL, //it doesn't matter through which node we upload content
+		srv.URL, //it doesn't matter through which node we upload content
 		"up",
 		"--encrypt",
 		dataFilename)
@@ -83,14 +76,14 @@ func TestAccessPassword(t *testing.T) {
 	}
 
 	ref := matches[0]
-
+	tmp, err := ioutil.TempDir("", "swarm-test")
-	password := "smth"
-	passwordFilename := filepath.Join(tmp, "password.txt")
-
-	err = ioutil.WriteFile(passwordFilename, []byte(password), 0666)
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(tmp)
+	password := "smth"
+	passwordFilename := testutil.TempFileWithContent(t, "smth")
+	defer os.RemoveAll(passwordFilename)
 
 	up = runSwarm(t,
 		"access",
@@ -142,8 +135,10 @@ func TestAccessPassword(t *testing.T) {
 	if a.KdfParams == nil {
 		t.Fatal("manifest access kdf params is nil")
 	}
-
+	if a.Publisher != "" {
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+		t.Fatal("should be empty")
+	}
+	client := swarm.NewClient(srv.URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
@@ -152,7 +147,7 @@ func TestAccessPassword(t *testing.T) {
 
 	httpClient := &http.Client{}
 
-	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
+	url := srv.URL + "/" + "bzz:/" + hash
 	response, err := httpClient.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -188,17 +183,13 @@ func TestAccessPassword(t *testing.T) {
 		t.Errorf("expected decrypted data %q, got %q", data, string(d))
 	}
 
-	wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt")
+	wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng")
-
+	defer os.RemoveAll(wrongPasswordFilename)
-	err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
 		"--bzzapi",
-		proxyNode.URL,
+		srv.URL,
 		"down",
 		"bzz:/"+hash,
 		tmp,
@@ -219,25 +210,11 @@ func TestAccessPassword(t *testing.T) {
 // the test will fail if the proxy's given private key is not granted on the ACT.
 func TestAccessPK(t *testing.T) {
 	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 1)
+	cluster := newTestCluster(t, 2)
 	defer cluster.Shutdown()
 
-	// create a tmp file
+	dataFilename := testutil.TempFileWithContent(t, data)
-	tmp, err := ioutil.TempFile("", "swarm-test")
+	defer os.RemoveAll(dataFilename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
-
-	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hashRegexp := `[a-f\d]{128}`
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
@@ -245,7 +222,7 @@ func TestAccessPK(t *testing.T) {
 		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
-		tmp.Name())
+		dataFilename)
 	_, matches := up.ExpectRegexp(hashRegexp)
 	up.ExpectExit()
 
@@ -254,7 +231,6 @@ func TestAccessPK(t *testing.T) {
 	}
 
 	ref := matches[0]
-
 	pk := cluster.Nodes[0].PrivateKey
 	granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
 
@@ -263,22 +239,15 @@ func TestAccessPK(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	passFile, err := ioutil.TempFile("", "swarm-test")
+	passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
-	if err != nil {
+	defer os.RemoveAll(passwordFilename)
-		t.Fatal(err)
+
-	}
-	defer passFile.Close()
-	defer os.Remove(passFile.Name())
-	_, err = io.WriteString(passFile, testPassphrase)
-	if err != nil {
-		t.Fatal(err)
-	}
 	_, publisherAccount := getTestAccount(t, publisherDir)
 	up = runSwarm(t,
 		"--bzzaccount",
 		publisherAccount.Address.String(),
 		"--password",
-		passFile.Name(),
+		passwordFilename,
 		"--datadir",
 		publisherDir,
 		"--bzzapi",
@@ -299,6 +268,20 @@ func TestAccessPK(t *testing.T) {
 		t.Fatalf("stdout not matched")
 	}
 
+	//get the public key from the publisher directory
+	publicKeyFromDataDir := runSwarm(t,
+		"--bzzaccount",
+		publisherAccount.Address.String(),
+		"--password",
+		passwordFilename,
+		"--datadir",
+		publisherDir,
+		"print-keys",
+		"--compressed",
+	)
+	_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
+	publicKeyFromDataDir.ExpectExit()
+	pkComp := strings.Split(publicKeyString[0], "=")[1]
 	var m api.Manifest
 
 	err = json.Unmarshal([]byte(matches[0]), &m)
@@ -332,7 +315,9 @@ func TestAccessPK(t *testing.T) {
 	if a.KdfParams != nil {
 		t.Fatal("manifest access kdf params should be nil")
 	}
-
+	if a.Publisher != pkComp {
+		t.Fatal("publisher key did not match")
+	}
 	client := swarm.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
@@ -359,36 +344,34 @@ func TestAccessPK(t *testing.T) {
 	}
 }
 
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control
+// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
-// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
-// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through
-// one of the nodes then disappears.
 func TestAccessACT(t *testing.T) {
+	testAccessACT(t, 0)
+}
+
+// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
+func TestAccessACTScale(t *testing.T) {
+	testAccessACT(t, 1000)
+}
+
+// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
+// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
+// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
+// the publisher uploads through one of the nodes then disappears.
+func testAccessACT(t *testing.T, bogusEntries int) {
 	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 3)
+	const clusterSize = 3
+	cluster := newTestCluster(t, clusterSize)
 	defer cluster.Shutdown()
 
 	var uploadThroughNode = cluster.Nodes[0]
 	client := swarm.NewClient(uploadThroughNode.URL)
 
 	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
-	nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`)
+	nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)
-	// create a tmp file
+	dataFilename := testutil.TempFileWithContent(t, data)
-	tmp, err := ioutil.TempFile("", "swarm-test")
+	defer os.RemoveAll(dataFilename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
-
-	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hashRegexp := `[a-f\d]{128}`
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
@@ -396,7 +379,7 @@ func TestAccessACT(t *testing.T) {
 		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
-		tmp.Name())
+		dataFilename)
 	_, matches := up.ExpectRegexp(hashRegexp)
 	up.ExpectExit()
 
@@ -415,41 +398,42 @@ func TestAccessACT(t *testing.T) {
 		grantees = append(grantees, hex.EncodeToString(granteePubKey))
 	}
 
-	granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv")
+	if bogusEntries > 0 {
+		bogusGrantees := []string{}
+
+		for i := 0; i < bogusEntries; i++ {
+			prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
-
+			bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey)))
-	_, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n"))
-	if err != nil {
-		t.Fatal(err)
 		}
-
+		r2 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
-	defer granteesPubkeyListFile.Close()
+		for i := 0; i < len(grantees); i++ {
-	defer os.Remove(granteesPubkeyListFile.Name())
+			insertAtIdx := r2.Intn(len(bogusGrantees))
+			bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...)
+		}
+		grantees = bogusGrantees
+	}
+	granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n"))
+	defer os.RemoveAll(granteesPubkeyListFile)
 
 	publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(publisherDir)
 
-	passFile, err := ioutil.TempFile("", "swarm-test")
+	passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
-	if err != nil {
+	defer os.RemoveAll(passwordFilename)
-		t.Fatal(err)
+	actPasswordFilename := testutil.TempFileWithContent(t, "smth")
-	}
+	defer os.RemoveAll(actPasswordFilename)
-	defer passFile.Close()
-	defer os.Remove(passFile.Name())
-	_, err = io.WriteString(passFile, testPassphrase)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	_, publisherAccount := getTestAccount(t, publisherDir)
 	up = runSwarm(t,
 		"--bzzaccount",
 		publisherAccount.Address.String(),
 		"--password",
-		passFile.Name(),
+		passwordFilename,
 		"--datadir",
 		publisherDir,
 		"--bzzapi",
@@ -458,7 +442,9 @@ func TestAccessACT(t *testing.T) {
 		"new",
 		"act",
 		"--grant-keys",
-		granteesPubkeyListFile.Name(),
+		granteesPubkeyListFile,
+		"--password",
+		actPasswordFilename,
 		ref,
 	)
 
@@ -468,6 +454,22 @@ func TestAccessACT(t *testing.T) {
 	if len(matches) == 0 {
 		t.Fatalf("stdout not matched")
 	}
+
+	//get the public key from the publisher directory
+	publicKeyFromDataDir := runSwarm(t,
+		"--bzzaccount",
+		publisherAccount.Address.String(),
+		"--password",
+		passwordFilename,
+		"--datadir",
+		publisherDir,
+		"print-keys",
+		"--compressed",
+	)
+	_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
+	publicKeyFromDataDir.ExpectExit()
+	pkComp := strings.Split(publicKeyString[0], "=")[1]
+
 	hash := matches[0]
 	m, _, err := client.DownloadManifest(hash)
 	if err != nil {
@@ -497,10 +499,10 @@ func TestAccessACT(t *testing.T) {
 	if len(a.Salt) < 32 {
 		t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
 	}
-	if a.KdfParams != nil {
-		t.Fatal("manifest access kdf params should be nil")
-	}
 
+	if a.Publisher != pkComp {
+		t.Fatal("publisher key did not match")
+	}
 	httpClient := &http.Client{}
 
 	// all nodes except the skipped node should be able to decrypt the content
@@ -521,6 +523,25 @@ func TestAccessACT(t *testing.T) {
 				t.Fatalf("should be a 401")
 			}
 
+			// try downloading using a password instead, using the unauthorized node
+			passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1)
+			response, err = httpClient.Get(passwordUrl)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if response.StatusCode != http.StatusOK {
+				t.Fatal("should be a 200")
+			}
+
+			// now try with the wrong password, expect 401
+			passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1)
+			response, err = httpClient.Get(passwordUrl)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if response.StatusCode != http.StatusUnauthorized {
+				t.Fatal("should be a 401")
+			}
 			continue
 		}
 

vendor/github.com/ethereum/go-ethereum/cmd/swarm/config.go

@@ -68,6 +68,7 @@ const (
 	SWARM_ENV_SWAP_API                = "SWARM_SWAP_API"
 	SWARM_ENV_SYNC_DISABLE            = "SWARM_SYNC_DISABLE"
 	SWARM_ENV_SYNC_UPDATE_DELAY       = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
 	SWARM_ENV_LIGHT_NODE_ENABLE       = "SWARM_LIGHT_NODE_ENABLE"
 	SWARM_ENV_DELIVERY_SKIP_CHECK     = "SWARM_DELIVERY_SKIP_CHECK"
 	SWARM_ENV_ENS_API                 = "SWARM_ENS_API"
@@ -79,6 +80,7 @@ const (
 	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
 	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 
@@ -124,7 +126,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
 	//get the account for the provided swarm account
 	prvkey := getAccount(config.BzzAccount, ctx, stack)
 	//set the resolved config path (geth --datadir)
-	config.Path = stack.InstanceDir()
+	config.Path = expandPath(stack.InstanceDir())
 	//finally, initialize the configuration
 	config.Init(prvkey)
 	//configuration phase completed here
@@ -175,14 +177,18 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 	}
 
 	if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
-		if id, _ := strconv.Atoi(networkid); id != 0 {
+		id, err := strconv.ParseUint(networkid, 10, 64)
-			currentConfig.NetworkID = uint64(id)
+		if err != nil {
+			utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err)
+		}
+		if id != 0 {
+			currentConfig.NetworkID = id
 		}
 	}
 
 	if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
 		if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" {
-			currentConfig.Path = datadir
+			currentConfig.Path = expandPath(datadir)
 		}
 	}
 
@@ -207,6 +213,9 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.SyncUpdateDelay = d
 	}
 
+	// any value including 0 is acceptable
+	currentConfig.MaxStreamPeerServers = ctx.GlobalInt(SwarmMaxStreamPeerServersFlag.Name)
+
 	if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
 		currentConfig.LightNodeEnabled = true
 	}
@@ -226,6 +235,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		if len(ensAPIs) == 1 && ensAPIs[0] == "" {
 			ensAPIs = nil
 		}
+		for i := range ensAPIs {
+			ensAPIs[i] = expandPath(ensAPIs[i])
+		}
+
 		currentConfig.EnsAPIs = ensAPIs
 	}
 
@@ -262,13 +275,17 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
-		if id, _ := strconv.Atoi(networkid); id != 0 {
+		id, err := strconv.ParseUint(networkid, 10, 64)
-			currentConfig.NetworkID = uint64(id)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
+		}
+		if id != 0 {
+			currentConfig.NetworkID = id
 		}
 	}
 
 	if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
-		currentConfig.Path = datadir
+		currentConfig.Path = expandPath(datadir)
 	}
 
 	bzzport := os.Getenv(SWARM_ENV_PORT)
@@ -281,33 +298,50 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
-		if swap, err := strconv.ParseBool(swapenable); err != nil {
+		swap, err := strconv.ParseBool(swapenable)
-			currentConfig.SwapEnabled = swap
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
 		}
+		currentConfig.SwapEnabled = swap
 	}
 
 	if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
-		if sync, err := strconv.ParseBool(syncdisable); err != nil {
+		sync, err := strconv.ParseBool(syncdisable)
-			currentConfig.SyncEnabled = !sync
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
 		}
+		currentConfig.SyncEnabled = !sync
 	}
 
 	if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
-		if skipCheck, err := strconv.ParseBool(v); err != nil {
+		skipCheck, err := strconv.ParseBool(v)
+		if err != nil {
 			currentConfig.DeliverySkipCheck = skipCheck
 		}
 	}
 
 	if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
-		if d, err := time.ParseDuration(v); err != nil {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
+		}
 		currentConfig.SyncUpdateDelay = d
 	}
+
+	if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
+		m, err := strconv.Atoi(max)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
+		}
+		currentConfig.MaxStreamPeerServers = m
 	}
 
 	if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
-		if lightnode, err := strconv.ParseBool(lne); err != nil {
+		lightnode, err := strconv.ParseBool(lne)
-			currentConfig.LightNodeEnabled = lightnode
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
 		}
+		currentConfig.LightNodeEnabled = lightnode
 	}
 
 	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {

vendor/github.com/ethereum/go-ethereum/cmd/swarm/db.go

@@ -29,6 +29,48 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var dbCommand = cli.Command{
+	Name:               "db",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "manage the local chunk database",
+	ArgsUsage:          "db COMMAND",
+	Description:        "Manage the local chunk database",
+	Subcommands: []cli.Command{
+		{
+			Action:             dbExport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "export",
+			Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `
+Export a local chunk database as a tar archive (use - to send to stdout).
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The export may be quite large, consider piping the output through the Unix
+pv(1) tool to get a progress bar:
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
+`,
+		},
+		{
+			Action:             dbImport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "import",
+			Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
+
+    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The import may be quite large, consider piping the input through the Unix
+pv(1) tool to get a progress bar:
+
+    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
+		},
+	},
+}
+
 func dbExport(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) != 3 {
@@ -93,21 +135,6 @@ func dbImport(ctx *cli.Context) {
 	log.Info(fmt.Sprintf("successfully imported %d chunks", count))
 }
 
-func dbClean(ctx *cli.Context) {
-	args := ctx.Args()
-	if len(args) != 2 {
-		utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
-	}
-
-	store, err := openLDBStore(args[0], common.Hex2Bytes(args[1]))
-	if err != nil {
-		utils.Fatalf("error opening local chunk database: %s", err)
-	}
-	defer store.Close()
-
-	store.Cleanup()
-}
-
 func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return nil, fmt.Errorf("invalid chunkdb path: %s", err)

vendor/github.com/ethereum/go-ethereum/cmd/swarm/download.go

@@ -28,6 +28,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var downloadCommand = cli.Command{
+	Action:      download,
+	Name:        "down",
+	Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
+	Usage:       "downloads a swarm manifest or a file inside a manifest",
+	ArgsUsage:   " <uri> [<dir>]",
+	Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
+}
+
 func download(ctx *cli.Context) {
 	log.Debug("downloading content using swarm down")
 	args := ctx.Args()

vendor/github.com/ethereum/go-ethereum/cmd/swarm/export_test.go

@@ -19,15 +19,15 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // TestCLISwarmExportImport perform the following test:
@@ -38,14 +38,18 @@ import (
 // 5. imports the exported datastore
 // 6. fetches the uploaded random file from the second node
 func TestCLISwarmExportImport(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	cluster := newTestCluster(t, 1)
 
 	// generate random 10mb file
-	f, cleanup := generateRandomFile(t, 10000000)
+	content := testutil.RandomBytes(1, 10000000)
-	defer cleanup()
+	fileName := testutil.TempFileWithContent(t, string(content))
+	defer os.Remove(fileName)
 
 	// upload the file with 'swarm up' and expect a hash
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
 	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
 	up.ExpectExit()
 	hash := matches[0]
@@ -92,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 
 	// compare downloaded file with the generated random file
-	mustEqualFiles(t, f, res.Body)
+	mustEqualFiles(t, bytes.NewReader(content), res.Body)
 }
 
 func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -113,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
 		t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
 	}
 }
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	// write 10mb random data to file
-	buf := make([]byte, 10000000)
-	_, err = rand.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
-}

vendor/github.com/ethereum/go-ethereum/cmd/swarm/feeds.go

@@ -0,0 +1,234 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command feed allows the user to create and update signed Swarm feeds
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var feedCommand = cli.Command{
+	CustomHelpTemplate: helpTemplate,
+	Name:               "feed",
+	Usage:              "(Advanced) Create and update Swarm Feeds",
+	ArgsUsage:          "<create|update|info>",
+	Description:        "Works with Swarm Feeds",
+	Subcommands: []cli.Command{
+		{
+			Action:             feedCreateManifest,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "create",
+			Usage:              "creates and publishes a new feed manifest",
+			Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+			Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+		{
+			Action:             feedUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "updates the content of an existing Swarm Feed",
+			ArgsUsage:          "<0x Hex data>",
+			Description: `publishes a new update on the specified topic
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the feed,
+					instead of using --topic / --name
+					`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
+		},
+		{
+			Action:             feedInfo,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "info",
+			Usage:              "obtains information about an existing Swarm feed",
+			Description: `obtains information about an existing Swarm feed
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the feed`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+	},
+}
+
+func NewGenericSigner(ctx *cli.Context) feed.Signer {
+	return feed.NewGenericSigner(getPrivKey(ctx))
+}
+
+func getTopic(ctx *cli.Context) (topic feed.Topic) {
+	var name = ctx.String(SwarmFeedNameFlag.Name)
+	var relatedTopic = ctx.String(SwarmFeedTopicFlag.Name)
+	var relatedTopicBytes []byte
+	var err error
+
+	if relatedTopic != "" {
+		relatedTopicBytes, err = hexutil.Decode(relatedTopic)
+		if err != nil {
+			utils.Fatalf("Error parsing topic: %s", err)
+		}
+	}
+
+	topic, err = feed.NewTopic(name, relatedTopicBytes)
+	if err != nil {
+		utils.Fatalf("Error parsing topic: %s", err)
+	}
+	return topic
+}
+
+// swarm feed create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
+// swarm feed update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
+// swarm feed info <Manifest Address or ENS domain>
+
+func feedCreateManifest(ctx *cli.Context) {
+	var (
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = swarm.NewClient(bzzapi)
+	)
+
+	newFeedUpdateRequest := feed.NewFirstRequest(getTopic(ctx))
+	newFeedUpdateRequest.Feed.User = feedGetUser(ctx)
+
+	manifestAddress, err := client.CreateFeedWithManifest(newFeedUpdateRequest)
+	if err != nil {
+		utils.Fatalf("Error creating feed manifest: %s", err.Error())
+		return
+	}
+	fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
+
+}
+
+func feedUpdate(ctx *cli.Context) {
+	args := ctx.Args()
+
+	var (
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
+	)
+
+	if len(args) < 1 {
+		fmt.Println("Incorrect number of arguments")
+		cli.ShowCommandHelpAndExit(ctx, "update", 1)
+		return
+	}
+
+	signer := NewGenericSigner(ctx)
+
+	data, err := hexutil.Decode(args[0])
+	if err != nil {
+		utils.Fatalf("Error parsing data: %s", err.Error())
+		return
+	}
+
+	var updateRequest *feed.Request
+	var query *feed.Query
+
+	if manifestAddressOrDomain == "" {
+		query = new(feed.Query)
+		query.User = signer.Address()
+		query.Topic = getTopic(ctx)
+
+	}
+
+	// Retrieve a feed update request
+	updateRequest, err = client.GetFeedRequest(query, manifestAddressOrDomain)
+	if err != nil {
+		utils.Fatalf("Error retrieving feed status: %s", err.Error())
+	}
+
+	// set the new data
+	updateRequest.SetData(data)
+
+	// sign update
+	if err = updateRequest.Sign(signer); err != nil {
+		utils.Fatalf("Error signing feed update: %s", err.Error())
+	}
+
+	// post update
+	err = client.UpdateFeed(updateRequest)
+	if err != nil {
+		utils.Fatalf("Error updating feed: %s", err.Error())
+		return
+	}
+}
+
+func feedInfo(ctx *cli.Context) {
+	var (
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
+	)
+
+	var query *feed.Query
+	if manifestAddressOrDomain == "" {
+		query = new(feed.Query)
+		query.Topic = getTopic(ctx)
+		query.User = feedGetUser(ctx)
+	}
+
+	metadata, err := client.GetFeedRequest(query, manifestAddressOrDomain)
+	if err != nil {
+		utils.Fatalf("Error retrieving feed metadata: %s", err.Error())
+		return
+	}
+	encodedMetadata, err := metadata.MarshalJSON()
+	if err != nil {
+		utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
+	}
+	fmt.Println(string(encodedMetadata))
+}
+
+func feedGetUser(ctx *cli.Context) common.Address {
+	var user = ctx.String(SwarmFeedUserFlag.Name)
+	if user != "" {
+		return common.HexToAddress(user)
+	}
+	pk := getPrivKey(ctx)
+	if pk == nil {
+		utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount")
+	}
+	return crypto.PubkeyToAddress(pk.PublicKey)
+
+}