common/hexutil: implement TextMarshaler, TextUnmarshaler

This commit makes the wrapper types more generally applicable.
encoding.TextMarshaler is supported by most codec implementations (e.g.
for yaml).

The tests now ensure that package json actually recognizes the custom
marshaler implementation irrespective of how it is implemented.

The Uint type has new tests, too. These are tricky because uint size
depends on the CPU word size. Turns out that there was one incorrect
case where decoding returned ErrUint64Range instead of ErrUintRange.

common/hexutil/hexutil_test.go

@@ -30,7 +30,8 @@ type marshalTest struct {
 type unmarshalTest struct {
 	input        string
 	want         interface{}
-	wantErr error
+	wantErr      error // if set, decoding must fail on any platform
+	wantErr32bit error // if set, decoding must fail on 32bit platforms (used for Uint tests)
 }
 
 var (
@@ -55,6 +56,13 @@ var (
 		{uint64(0x1122334455667788), "0x1122334455667788"},
 	}
 
+	encodeUintTests = []marshalTest{
+		{uint(0), "0x0"},
+		{uint(1), "0x1"},
+		{uint(0xff), "0xff"},
+		{uint(0x11223344), "0x11223344"},
+	}
+
 	decodeBytesTests = []unmarshalTest{
 		// invalid
 		{input: ``, wantErr: ErrEmptyString},

common/hexutil/json.go

@@ -25,8 +25,7 @@ import (
 )
 
 var (
-	jsonNull          = []byte("null")
+	textZero          = []byte(`0x0`)
-	jsonZero          = []byte(`"0x0"`)
 	errNonString      = errors.New("cannot unmarshal non-string as hex data")
 	errNegativeBigInt = errors.New("hexutil.Big: can't marshal negative integer")
 )
@@ -35,18 +34,25 @@ var (
 // The empty slice marshals as "0x".
 type Bytes []byte
 
-// MarshalJSON implements json.Marshaler.
+// MarshalText implements encoding.TextMarshaler
-func (b Bytes) MarshalJSON() ([]byte, error) {
+func (b Bytes) MarshalText() ([]byte, error) {
-	result := make([]byte, len(b)*2+4)
+	result := make([]byte, len(b)*2+2)
-	copy(result, `"0x`)
+	copy(result, `0x`)
-	hex.Encode(result[3:], b)
+	hex.Encode(result[2:], b)
-	result[len(result)-1] = '"'
 	return result, nil
 }
 
 // UnmarshalJSON implements json.Unmarshaler.
 func (b *Bytes) UnmarshalJSON(input []byte) error {
-	raw, err := checkJSON(input)
+	if !isString(input) {
+		return errNonString
+	}
+	return b.UnmarshalText(input[1 : len(input)-1])
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (b *Bytes) UnmarshalText(input []byte) error {
+	raw, err := checkText(input)
 	if err != nil {
 		return err
 	}
@@ -64,17 +70,11 @@ func (b Bytes) String() string {
 	return Encode(b)
 }
 
-// UnmarshalJSON decodes input as a JSON string with 0x prefix. The length of out
+// UnmarshalFixedText decodes the input as a string with 0x prefix. The length of out
 // determines the required input length. This function is commonly used to implement the
-// UnmarshalJSON method for fixed-size types:
+// UnmarshalText method for fixed-size types.
-//
+func UnmarshalFixedText(typname string, input, out []byte) error {
-//     type Foo [8]byte
+	raw, err := checkText(input)
-//
-//     func (f *Foo) UnmarshalJSON(input []byte) error {
-//         return hexutil.UnmarshalJSON("Foo", input, f[:])
-//     }
-func UnmarshalJSON(typname string, input, out []byte) error {
-	raw, err := checkJSON(input)
 	if err != nil {
 		return err
 	}
@@ -99,26 +99,33 @@ func UnmarshalJSON(typname string, input, out []byte) error {
 // marshaled without error.
 type Big big.Int
 
-// MarshalJSON implements json.Marshaler.
+// MarshalText implements encoding.TextMarshaler
-func (b *Big) MarshalJSON() ([]byte, error) {
+func (b Big) MarshalText() ([]byte, error) {
-	if b == nil {
+	bigint := (big.Int)(b)
-		return jsonNull, nil
-	}
-	bigint := (*big.Int)(b)
 	if bigint.Sign() == -1 {
 		return nil, errNegativeBigInt
 	}
 	nbits := bigint.BitLen()
 	if nbits == 0 {
-		return jsonZero, nil
+		return textZero, nil
 	}
-	enc := fmt.Sprintf(`"0x%x"`, bigint)
+	enc := make([]byte, 2, nbits/4+2)
-	return []byte(enc), nil
+	copy(enc, "0x")
+	enc = bigint.Append(enc, 16)
+	return enc, nil
 }
 
 // UnmarshalJSON implements json.Unmarshaler.
 func (b *Big) UnmarshalJSON(input []byte) error {
-	raw, err := checkNumberJSON(input)
+	if !isString(input) {
+		return errNonString
+	}
+	return b.UnmarshalText(input[1 : len(input)-1])
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler
+func (b *Big) UnmarshalText(input []byte) error {
+	raw, err := checkNumberText(input)
 	if err != nil {
 		return err
 	}
@@ -162,18 +169,25 @@ func (b *Big) String() string {
 // The zero value marshals as "0x0".
 type Uint64 uint64
 
-// MarshalJSON implements json.Marshaler.
+// MarshalText implements encoding.TextMarshaler.
-func (b Uint64) MarshalJSON() ([]byte, error) {
+func (b Uint64) MarshalText() ([]byte, error) {
-	buf := make([]byte, 3, 12)
+	buf := make([]byte, 2, 10)
-	copy(buf, `"0x`)
+	copy(buf, `0x`)
 	buf = strconv.AppendUint(buf, uint64(b), 16)
-	buf = append(buf, '"')
 	return buf, nil
 }
 
 // UnmarshalJSON implements json.Unmarshaler.
 func (b *Uint64) UnmarshalJSON(input []byte) error {
-	raw, err := checkNumberJSON(input)
+	if !isString(input) {
+		return errNonString
+	}
+	return b.UnmarshalText(input[1 : len(input)-1])
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler
+func (b *Uint64) UnmarshalText(input []byte) error {
+	raw, err := checkNumberText(input)
 	if err != nil {
 		return err
 	}
@@ -202,19 +216,27 @@ func (b Uint64) String() string {
 // The zero value marshals as "0x0".
 type Uint uint
 
-// MarshalJSON implements json.Marshaler.
+// MarshalText implements encoding.TextMarshaler.
-func (b Uint) MarshalJSON() ([]byte, error) {
+func (b Uint) MarshalText() ([]byte, error) {
-	return Uint64(b).MarshalJSON()
+	return Uint64(b).MarshalText()
 }
 
 // UnmarshalJSON implements json.Unmarshaler.
 func (b *Uint) UnmarshalJSON(input []byte) error {
+	if !isString(input) {
+		return errNonString
+	}
+	return b.UnmarshalText(input[1 : len(input)-1])
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (b *Uint) UnmarshalText(input []byte) error {
 	var u64 Uint64
-	err := u64.UnmarshalJSON(input)
+	err := u64.UnmarshalText(input)
-	if err != nil {
+	if u64 > Uint64(^uint(0)) || err == ErrUint64Range {
-		return err
-	} else if u64 > Uint64(^uint(0)) {
 		return ErrUintRange
+	} else if err != nil {
+		return err
 	}
 	*b = Uint(u64)
 	return nil
@@ -233,28 +255,21 @@ func bytesHave0xPrefix(input []byte) bool {
 	return len(input) >= 2 && input[0] == '0' && (input[1] == 'x' || input[1] == 'X')
 }
 
-func checkJSON(input []byte) (raw []byte, err error) {
+func checkText(input []byte) ([]byte, error) {
-	if !isString(input) {
+	if len(input) == 0 {
-		return nil, errNonString
-	}
-	if len(input) == 2 {
 		return nil, nil // empty strings are allowed
 	}
-	if !bytesHave0xPrefix(input[1:]) {
+	if !bytesHave0xPrefix(input) {
 		return nil, ErrMissingPrefix
 	}
-	input = input[3 : len(input)-1]
+	input = input[2:]
 	if len(input)%2 != 0 {
 		return nil, ErrOddLength
 	}
 	return input, nil
 }
 
-func checkNumberJSON(input []byte) (raw []byte, err error) {
+func checkNumberText(input []byte) (raw []byte, err error) {
-	if !isString(input) {
-		return nil, errNonString
-	}
-	input = input[1 : len(input)-1]
 	if len(input) == 0 {
 		return nil, nil // empty strings are allowed
 	}

common/hexutil/json_example_test.go

@@ -0,0 +1,45 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package hexutil_test
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+type MyType [5]byte
+
+func (v *MyType) UnmarshalText(input []byte) error {
+	return hexutil.UnmarshalFixedText("MyType", input, v[:])
+}
+
+func (v MyType) String() string {
+	return hexutil.Bytes(v[:]).String()
+}
+
+func ExampleUnmarshalFixedText() {
+	var v1, v2 MyType
+	fmt.Println("v1 error:", json.Unmarshal([]byte(`"0x01"`), &v1))
+	fmt.Println("v2 error:", json.Unmarshal([]byte(`"0x0101010101"`), &v2))
+	fmt.Println("v2:", v2)
+	// Output:
+	// v1 error: hex string has length 2, want 10 for MyType
+	// v2 error: <nil>
+	// v2: 0x0101010101
+}

common/hexutil/json_test.go

@@ -19,6 +19,8 @@ package hexutil
 import (
 	"bytes"
 	"encoding/hex"
+	"encoding/json"
+	"errors"
 	"math/big"
 	"testing"
 )
@@ -55,9 +57,11 @@ func referenceBytes(s string) []byte {
 	return b
 }
 
+var errJSONEOF = errors.New("unexpected end of JSON input")
+
 var unmarshalBytesTests = []unmarshalTest{
 	// invalid encoding
-	{input: "", wantErr: errNonString},
+	{input: "", wantErr: errJSONEOF},
 	{input: "null", wantErr: errNonString},
 	{input: "10", wantErr: errNonString},
 	{input: `"0"`, wantErr: ErrMissingPrefix},
@@ -80,7 +84,7 @@ var unmarshalBytesTests = []unmarshalTest{
 func TestUnmarshalBytes(t *testing.T) {
 	for _, test := range unmarshalBytesTests {
 		var v Bytes
-		err := v.UnmarshalJSON([]byte(test.input))
+		err := json.Unmarshal([]byte(test.input), &v)
 		if !checkError(t, test.input, err, test.wantErr) {
 			continue
 		}
@@ -104,7 +108,7 @@ func BenchmarkUnmarshalBytes(b *testing.B) {
 func TestMarshalBytes(t *testing.T) {
 	for _, test := range encodeBytesTests {
 		in := test.input.([]byte)
-		out, err := Bytes(in).MarshalJSON()
+		out, err := json.Marshal(Bytes(in))
 		if err != nil {
 			t.Errorf("%x: %v", in, err)
 			continue
@@ -122,7 +126,7 @@ func TestMarshalBytes(t *testing.T) {
 
 var unmarshalBigTests = []unmarshalTest{
 	// invalid encoding
-	{input: "", wantErr: errNonString},
+	{input: "", wantErr: errJSONEOF},
 	{input: "null", wantErr: errNonString},
 	{input: "10", wantErr: errNonString},
 	{input: `"0"`, wantErr: ErrMissingPrefix},
@@ -161,7 +165,7 @@ var unmarshalBigTests = []unmarshalTest{
 func TestUnmarshalBig(t *testing.T) {
 	for _, test := range unmarshalBigTests {
 		var v Big
-		err := v.UnmarshalJSON([]byte(test.input))
+		err := json.Unmarshal([]byte(test.input), &v)
 		if !checkError(t, test.input, err, test.wantErr) {
 			continue
 		}
@@ -185,7 +189,7 @@ func BenchmarkUnmarshalBig(b *testing.B) {
 func TestMarshalBig(t *testing.T) {
 	for _, test := range encodeBigTests {
 		in := test.input.(*big.Int)
-		out, err := (*Big)(in).MarshalJSON()
+		out, err := json.Marshal((*Big)(in))
 		if err != nil {
 			t.Errorf("%d: %v", in, err)
 			continue
@@ -203,7 +207,7 @@ func TestMarshalBig(t *testing.T) {
 
 var unmarshalUint64Tests = []unmarshalTest{
 	// invalid encoding
-	{input: "", wantErr: errNonString},
+	{input: "", wantErr: errJSONEOF},
 	{input: "null", wantErr: errNonString},
 	{input: "10", wantErr: errNonString},
 	{input: `"0"`, wantErr: ErrMissingPrefix},
@@ -227,7 +231,7 @@ var unmarshalUint64Tests = []unmarshalTest{
 func TestUnmarshalUint64(t *testing.T) {
 	for _, test := range unmarshalUint64Tests {
 		var v Uint64
-		err := v.UnmarshalJSON([]byte(test.input))
+		err := json.Unmarshal([]byte(test.input), &v)
 		if !checkError(t, test.input, err, test.wantErr) {
 			continue
 		}
@@ -249,7 +253,7 @@ func BenchmarkUnmarshalUint64(b *testing.B) {
 func TestMarshalUint64(t *testing.T) {
 	for _, test := range encodeUint64Tests {
 		in := test.input.(uint64)
-		out, err := Uint64(in).MarshalJSON()
+		out, err := json.Marshal(Uint64(in))
 		if err != nil {
 			t.Errorf("%d: %v", in, err)
 			continue
@@ -264,3 +268,72 @@ func TestMarshalUint64(t *testing.T) {
 		}
 	}
 }
+
+func TestMarshalUint(t *testing.T) {
+	for _, test := range encodeUintTests {
+		in := test.input.(uint)
+		out, err := json.Marshal(Uint(in))
+		if err != nil {
+			t.Errorf("%d: %v", in, err)
+			continue
+		}
+		if want := `"` + test.want + `"`; string(out) != want {
+			t.Errorf("%d: MarshalJSON output mismatch: got %q, want %q", in, out, want)
+			continue
+		}
+		if out := (Uint)(in).String(); out != test.want {
+			t.Errorf("%x: String mismatch: got %q, want %q", in, out, test.want)
+			continue
+		}
+	}
+}
+
+var (
+	// These are variables (not constants) to avoid constant overflow
+	// checks in the compiler on 32bit platforms.
+	maxUint33bits = uint64(^uint32(0)) + 1
+	maxUint64bits = ^uint64(0)
+)
+
+var unmarshalUintTests = []unmarshalTest{
+	// invalid encoding
+	{input: "", wantErr: errJSONEOF},
+	{input: "null", wantErr: errNonString},
+	{input: "10", wantErr: errNonString},
+	{input: `"0"`, wantErr: ErrMissingPrefix},
+	{input: `"0x"`, wantErr: ErrEmptyNumber},
+	{input: `"0x01"`, wantErr: ErrLeadingZero},
+	{input: `"0x100000000"`, want: uint(maxUint33bits), wantErr32bit: ErrUintRange},
+	{input: `"0xfffffffffffffffff"`, wantErr: ErrUintRange},
+	{input: `"0xx"`, wantErr: ErrSyntax},
+	{input: `"0x1zz01"`, wantErr: ErrSyntax},
+
+	// valid encoding
+	{input: `""`, want: uint(0)},
+	{input: `"0x0"`, want: uint(0)},
+	{input: `"0x2"`, want: uint(0x2)},
+	{input: `"0x2F2"`, want: uint(0x2f2)},
+	{input: `"0X2F2"`, want: uint(0x2f2)},
+	{input: `"0x1122aaff"`, want: uint(0x1122aaff)},
+	{input: `"0xbbb"`, want: uint(0xbbb)},
+	{input: `"0xffffffff"`, want: uint(0xffffffff)},
+	{input: `"0xffffffffffffffff"`, want: uint(maxUint64bits), wantErr32bit: ErrUintRange},
+}
+
+func TestUnmarshalUint(t *testing.T) {
+	for _, test := range unmarshalUintTests {
+		var v Uint
+		err := json.Unmarshal([]byte(test.input), &v)
+		if uintBits == 32 && test.wantErr32bit != nil {
+			checkError(t, test.input, err, test.wantErr32bit)
+			continue
+		}
+		if !checkError(t, test.input, err, test.wantErr) {
+			continue
+		}
+		if uint(v) != test.want.(uint) {
+			t.Errorf("input %s: value mismatch: got %d, want %d", test.input, v, test.want)
+			continue
+		}
+	}
+}

common/types.go

@@ -71,14 +71,14 @@ func (h Hash) Format(s fmt.State, c rune) {
 	fmt.Fprintf(s, "%"+string(c), h[:])
 }
 
-// UnmarshalJSON parses a hash in its hex from to a hash.
+// UnmarshalText parses a hash in hex syntax.
-func (h *Hash) UnmarshalJSON(input []byte) error {
+func (h *Hash) UnmarshalText(input []byte) error {
-	return hexutil.UnmarshalJSON("Hash", input, h[:])
+	return hexutil.UnmarshalFixedText("Hash", input, h[:])
 }
 
-// Serialize given hash to JSON
+// MarshalText returns the hex representation of h.
-func (h Hash) MarshalJSON() ([]byte, error) {
+func (h Hash) MarshalText() ([]byte, error) {
-	return hexutil.Bytes(h[:]).MarshalJSON()
+	return hexutil.Bytes(h[:]).MarshalText()
 }
 
 // Sets the hash to the value of b. If b is larger than len(h) it will panic
@@ -171,14 +171,14 @@ func (a *Address) Set(other Address) {
 	}
 }
 
-// Serialize given address to JSON
+// MarshalText returns the hex representation of a.
-func (a Address) MarshalJSON() ([]byte, error) {
+func (a Address) MarshalText() ([]byte, error) {
-	return hexutil.Bytes(a[:]).MarshalJSON()
+	return hexutil.Bytes(a[:]).MarshalText()
 }
 
-// Parse address from raw json data
+// UnmarshalText parses a hash in hex syntax.
-func (a *Address) UnmarshalJSON(input []byte) error {
+func (a *Address) UnmarshalText(input []byte) error {
-	return hexutil.UnmarshalJSON("Address", input, a[:])
+	return hexutil.UnmarshalFixedText("Address", input, a[:])
 }
 
 // PP Pretty Prints a byte slice in the following format:

common/types_test.go

@@ -17,6 +17,7 @@
 package common
 
 import (
+	"encoding/json"
 	"math/big"
 	"strings"
 	"testing"
@@ -37,7 +38,6 @@ func TestBytesConversion(t *testing.T) {
 }
 
 func TestHashJsonValidation(t *testing.T) {
-	var h Hash
 	var tests = []struct {
 		Prefix string
 		Size   int
@@ -52,7 +52,8 @@ func TestHashJsonValidation(t *testing.T) {
 	}
 	for _, test := range tests {
 		input := `"` + test.Prefix + strings.Repeat("0", test.Size) + `"`
-		err := h.UnmarshalJSON([]byte(input))
+		var v Hash
+		err := json.Unmarshal([]byte(input), &v)
 		if err == nil {
 			if test.Error != "" {
 				t.Errorf("%s: error mismatch: have nil, want %q", input, test.Error)
@@ -66,7 +67,6 @@ func TestHashJsonValidation(t *testing.T) {
 }
 
 func TestAddressUnmarshalJSON(t *testing.T) {
-	var a Address
 	var tests = []struct {
 		Input     string
 		ShouldErr bool
@@ -81,7 +81,8 @@ func TestAddressUnmarshalJSON(t *testing.T) {
 		{`"0x0000000000000000000000000000000000000010"`, false, big.NewInt(16)},
 	}
 	for i, test := range tests {
-		err := a.UnmarshalJSON([]byte(test.Input))
+		var v Address
+		err := json.Unmarshal([]byte(test.Input), &v)
 		if err != nil && !test.ShouldErr {
 			t.Errorf("test #%d: unexpected error: %v", i, err)
 		}
@@ -89,8 +90,8 @@ func TestAddressUnmarshalJSON(t *testing.T) {
 			if test.ShouldErr {
 				t.Errorf("test #%d: expected error, got none", i)
 			}
-			if a.Big().Cmp(test.Output) != 0 {
+			if v.Big().Cmp(test.Output) != 0 {
-				t.Errorf("test #%d: address mismatch: have %v, want %v", i, a.Big(), test.Output)
+				t.Errorf("test #%d: address mismatch: have %v, want %v", i, v.Big(), test.Output)
 			}
 		}
 	}

core/types/block.go

@@ -62,14 +62,14 @@ func (n BlockNonce) Uint64() uint64 {
 	return binary.BigEndian.Uint64(n[:])
 }
 
-// MarshalJSON implements json.Marshaler
+// MarshalText encodes n as a hex string with 0x prefix.
-func (n BlockNonce) MarshalJSON() ([]byte, error) {
+func (n BlockNonce) MarshalText() ([]byte, error) {
-	return hexutil.Bytes(n[:]).MarshalJSON()
+	return hexutil.Bytes(n[:]).MarshalText()
 }
 
-// UnmarshalJSON implements json.Unmarshaler
+// UnmarshalText implements encoding.TextUnmarshaler.
-func (n *BlockNonce) UnmarshalJSON(input []byte) error {
+func (n *BlockNonce) UnmarshalText(input []byte) error {
-	return hexutil.UnmarshalJSON("BlockNonce", input, n[:])
+	return hexutil.UnmarshalFixedText("BlockNonce", input, n[:])
 }
 
 // Header represents a block header in the Ethereum blockchain.

core/types/bloom9.go

@@ -75,14 +75,14 @@ func (b Bloom) TestBytes(test []byte) bool {
 
 }
 
-// MarshalJSON encodes b as a hex string with 0x prefix.
+// MarshalText encodes b as a hex string with 0x prefix.
-func (b Bloom) MarshalJSON() ([]byte, error) {
+func (b Bloom) MarshalText() ([]byte, error) {
-	return hexutil.Bytes(b[:]).MarshalJSON()
+	return hexutil.Bytes(b[:]).MarshalText()
 }
 
-// UnmarshalJSON b as a hex string with 0x prefix.
+// UnmarshalText b as a hex string with 0x prefix.
-func (b *Bloom) UnmarshalJSON(input []byte) error {
+func (b *Bloom) UnmarshalText(input []byte) error {
-	return hexutil.UnmarshalJSON("Bloom", input, b[:])
+	return hexutil.UnmarshalFixedText("Bloom", input, b[:])
 }
 
 func CreateBloom(receipts Receipts) Bloom {