Allow Parity as ingest node (#36)

* Upgrade go-ethereum to v1.8 * Add Node Info for parity nodes * Upgrade start_private_blockchain to use v1.8
2018-03-07 15:29:21 -06:00
parent 203f9b47d3
commit 5a652190d9
624 changed files with 77003 additions and 14158 deletions
@@ -25,6 +25,6 @@ var (
 	Big3   = big.NewInt(3)
 	Big0   = big.NewInt(0)
 	Big32  = big.NewInt(32)
-	Big256 = big.NewInt(0xff)
+	Big256 = big.NewInt(256)
 	Big257 = big.NewInt(257)
 )
@@ -40,7 +40,7 @@ func fastXORBytes(dst, a, b []byte) int {
 			dw[i] = aw[i] ^ bw[i]
 		}
 	}
-	for i := (n - n%wordSize); i < n; i++ {
+	for i := n - n%wordSize; i < n; i++ {
 		dst[i] = a[i] ^ b[i]
 	}
 	return n
@@ -84,7 +84,7 @@ func fastANDBytes(dst, a, b []byte) int {
 			dw[i] = aw[i] & bw[i]
 		}
 	}
-	for i := (n - n%wordSize); i < n; i++ {
+	for i := n - n%wordSize; i < n; i++ {
 		dst[i] = a[i] & b[i]
 	}
 	return n
@@ -128,7 +128,7 @@ func fastORBytes(dst, a, b []byte) int {
 			dw[i] = aw[i] | bw[i]
 		}
 	}
-	for i := (n - n%wordSize); i < n; i++ {
+	for i := n - n%wordSize; i < n; i++ {
 		dst[i] = a[i] | b[i]
 	}
 	return n
@@ -168,7 +168,7 @@ func fastTestBytes(p []byte) bool {
 			}
 		}
 	}
-	for i := (n - n%wordSize); i < n; i++ {
+	for i := n - n%wordSize; i < n; i++ {
 		if p[i] != 0 {
 			return true
 		}
@@ -17,9 +17,7 @@
 // Package common contains various helper functions.
 package common

-import (
-	"encoding/hex"
-)
+import "encoding/hex"

 func ToHex(b []byte) string {
 	hex := Bytes2Hex(b)
@@ -35,12 +33,11 @@ func FromHex(s string) []byte {
 		if s[0:2] == "0x" || s[0:2] == "0X" {
 			s = s[2:]
 		}
-		if len(s)%2 == 1 {
-			s = "0" + s
-		}
-		return Hex2Bytes(s)
 	}
-	return nil
+	if len(s)%2 == 1 {
+		s = "0" + s
+	}
+	return Hex2Bytes(s)
 }

 // Copy bytes
@@ -56,14 +53,24 @@ func CopyBytes(b []byte) (copiedBytes []byte) {
 	return
 }

-func HasHexPrefix(str string) bool {
-	l := len(str)
-	return l >= 2 && str[0:2] == "0x"
+func hasHexPrefix(str string) bool {
+	return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')
 }

-func IsHex(str string) bool {
-	l := len(str)
-	return l >= 4 && l%2 == 0 && str[0:2] == "0x"
+func isHexCharacter(c byte) bool {
+	return ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')
+}
+
+func isHex(str string) bool {
+	if len(str)%2 != 0 {
+		return false
+	}
+	for _, c := range []byte(str) {
+		if !isHexCharacter(c) {
+			return false
+		}
+	}
+	return true
 }

 func Bytes2Hex(d []byte) string {
@@ -34,19 +34,6 @@ func (s *BytesSuite) TestCopyBytes(c *checker.C) {
 	c.Assert(res1, checker.DeepEquals, exp1)
 }

-func (s *BytesSuite) TestIsHex(c *checker.C) {
-	data1 := "a9e67e"
-	exp1 := false
-	res1 := IsHex(data1)
-	c.Assert(res1, checker.DeepEquals, exp1)
-
-	data2 := "0xa9e67e00"
-	exp2 := true
-	res2 := IsHex(data2)
-	c.Assert(res2, checker.DeepEquals, exp2)
-
-}
-
 func (s *BytesSuite) TestLeftPadBytes(c *checker.C) {
 	val1 := []byte{1, 2, 3, 4}
 	exp1 := []byte{0, 0, 0, 0, 1, 2, 3, 4}
@@ -74,7 +61,28 @@ func TestFromHex(t *testing.T) {
 	expected := []byte{1}
 	result := FromHex(input)
 	if !bytes.Equal(expected, result) {
-		t.Errorf("Expected % x got % x", expected, result)
+		t.Errorf("Expected %x got %x", expected, result)
+	}
+}
+
+func TestIsHex(t *testing.T) {
+	tests := []struct {
+		input string
+		ok    bool
+	}{
+		{"", true},
+		{"0", false},
+		{"00", true},
+		{"a9e67e", true},
+		{"A9E67E", true},
+		{"0xa9e67e", false},
+		{"a9e67e001", false},
+		{"0xHELLO_MY_NAME_IS_STEVEN_@#$^&*", false},
+	}
+	for _, test := range tests {
+		if ok := isHex(test.input); ok != test.ok {
+			t.Errorf("isHex(%q) = %v, want %v", test.input, ok, test.ok)
+		}
 	}
 }

@@ -83,6 +91,15 @@ func TestFromHexOddLength(t *testing.T) {
 	expected := []byte{1}
 	result := FromHex(input)
 	if !bytes.Equal(expected, result) {
-		t.Errorf("Expected % x got % x", expected, result)
+		t.Errorf("Expected %x got %x", expected, result)
+	}
+}
+
+func TestNoPrefixShortHexOddLength(t *testing.T) {
+	input := "1"
+	expected := []byte{1}
+	result := FromHex(input)
+	if !bytes.Equal(expected, result) {
+		t.Errorf("Expected %x got %x", expected, result)
 	}
 }
@@ -0,0 +1,64 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build freebsd
+
+package fdlimit
+
+import "syscall"
+
+// This file is largely identical to fdlimit_unix.go,
+// but Rlimit fields have type int64 on FreeBSD so it needs
+// an extra conversion.
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func Raise(max uint64) error {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > int64(max) {
+		limit.Cur = int64(max)
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Max), nil
+}
@@ -0,0 +1,45 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestFileDescriptorLimits simply tests whether the file descriptor allowance
+// per this process can be retrieved.
+func TestFileDescriptorLimits(t *testing.T) {
+	target := 4096
+	hardlimit, err := Maximum()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hardlimit < target {
+		t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target))
+	}
+
+	if limit, err := Current(); err != nil || limit <= 0 {
+		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
+	}
+	if err := Raise(uint64(target)); err != nil {
+		t.Fatalf("failed to raise file allowance")
+	}
+	if limit, err := Current(); err != nil || limit < target {
+		t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err)
+	}
+}
@@ -0,0 +1,60 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build linux darwin netbsd openbsd solaris
+
+package fdlimit
+
+import "syscall"
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func Raise(max uint64) error {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Max), nil
+}
@@ -0,0 +1,47 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import "errors"
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+func Raise(max uint64) error {
+	// This method is NOP by design:
+	//  * Linux/Darwin counterparts need to manually increase per process limits
+	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
+	//    changeable from within a running process
+	// This way we can always "request" raising the limits, which will either have
+	// or not have effect based on the platform we're running on.
+	if max > 16384 {
+		return errors.New("file descriptor limit (16384) reached")
+	}
+	return nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	// Please see Raise for the reason why we use hard coded 16K as the limit
+	return 16384, nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	return Current()
+}
@@ -20,18 +20,29 @@ import (
 	"fmt"
 )

+// StorageSize is a wrapper around a float value that supports user friendly
+// formatting.
 type StorageSize float64

-func (self StorageSize) String() string {
-	if self > 1000000 {
-		return fmt.Sprintf("%.2f mB", self/1000000)
-	} else if self > 1000 {
-		return fmt.Sprintf("%.2f kB", self/1000)
+// String implements the stringer interface.
+func (s StorageSize) String() string {
+	if s > 1000000 {
+		return fmt.Sprintf("%.2f mB", s/1000000)
+	} else if s > 1000 {
+		return fmt.Sprintf("%.2f kB", s/1000)
 	} else {
-		return fmt.Sprintf("%.2f B", self)
+		return fmt.Sprintf("%.2f B", s)
 	}
 }

-func (self StorageSize) Int64() int64 {
-	return int64(self)
+// TerminalString implements log.TerminalStringer, formatting a string for console
+// output during logging.
+func (s StorageSize) TerminalString() string {
+	if s > 1000000 {
+		return fmt.Sprintf("%.2fmB", s/1000000)
+	} else if s > 1000 {
+		return fmt.Sprintf("%.2fkB", s/1000)
+	} else {
+		return fmt.Sprintf("%.2fB", s)
+	}
 }
@@ -150,13 +150,10 @@ func HexToAddress(s string) Address    { return BytesToAddress(FromHex(s)) }
 // IsHexAddress verifies whether a string can represent a valid hex-encoded
 // Ethereum address or not.
 func IsHexAddress(s string) bool {
-	if len(s) == 2+2*AddressLength && IsHex(s) {
-		return true
+	if hasHexPrefix(s) {
+		s = s[2:]
 	}
-	if len(s) == 2*AddressLength && IsHex("0x"+s) {
-		return true
-	}
-	return false
+	return len(s) == 2*AddressLength && isHex(s)
 }

 // Get the string representation of the underlying address
@@ -35,6 +35,30 @@ func TestBytesConversion(t *testing.T) {
 	}
 }

+func TestIsHexAddress(t *testing.T) {
+	tests := []struct {
+		str string
+		exp bool
+	}{
+		{"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", true},
+		{"5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", true},
+		{"0X5aaeb6053f3e94c9b9a09f33669435e7ef1beaed", true},
+		{"0XAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", true},
+		{"0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", true},
+		{"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed1", false},
+		{"0x5aaeb6053f3e94c9b9a09f33669435e7ef1beae", false},
+		{"5aaeb6053f3e94c9b9a09f33669435e7ef1beaed11", false},
+		{"0xxaaeb6053f3e94c9b9a09f33669435e7ef1beaed", false},
+	}
+
+	for _, test := range tests {
+		if result := IsHexAddress(test.str); result != test.exp {
+			t.Errorf("IsHexAddress(%s) == %v; expected %v",
+				test.str, result, test.exp)
+		}
+	}
+}
+
 func TestHashJsonValidation(t *testing.T) {
 	var tests = []struct {
 		Prefix string