Merge pull request #1484 from fjl/rlp-json-tests

cmd/ethtest, tests: add support for RLP JSON tests

cmd/ethtest/main.go

@@ -35,7 +35,7 @@ var (
 	testExtension   = ".json"
 	defaultTest     = "all"
 	defaultDir      = "."
-	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests"}
+	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"}
 	skipTests       = []string{}
 
 	TestFlag = cli.StringFlag{
@@ -75,6 +75,8 @@ func runTestWithReader(test string, r io.Reader) error {
 		err = tests.RunTransactionTestsWithReader(r, skipTests)
 	case "vm", "vmtest", "vmtests":
 		err = tests.RunVmTestWithReader(r, skipTests)
+	case "rlp", "rlptest", "rlptests":
+		err = tests.RunRLPTestWithReader(r, skipTests)
 	default:
 		err = fmt.Errorf("Invalid test type specified: %v", test)
 	}

rlp/decode.go

@@ -110,9 +110,17 @@ func Decode(r io.Reader, val interface{}) error {
 
 // DecodeBytes parses RLP data from b into val.
 // Please see the documentation of Decode for the decoding rules.
+// The input must contain exactly one value and no trailing data.
 func DecodeBytes(b []byte, val interface{}) error {
 	// TODO: this could use a Stream from a pool.
-	return NewStream(bytes.NewReader(b), uint64(len(b))).Decode(val)
+	r := bytes.NewReader(b)
+	if err := NewStream(r, uint64(len(b))).Decode(val); err != nil {
+		return err
+	}
+	if r.Len() > 0 {
+		return ErrMoreThanOneValue
+	}
+	return nil
 }
 
 type decodeError struct {
@@ -353,7 +361,7 @@ func decodeByteArray(s *Stream, val reflect.Value) error {
 			return err
 		}
 		// Reject cases where single byte encoding should have been used.
-		if size == 1 && slice[0] < 56 {
+		if size == 1 && slice[0] < 128 {
 			return wrapStreamError(ErrCanonSize, val.Type())
 		}
 	case List:
@@ -517,6 +525,10 @@ var (
 	ErrElemTooLarge   = errors.New("rlp: element is larger than containing list")
 	ErrValueTooLarge  = errors.New("rlp: value size exceeds available input length")
 
+	// This error is reported by DecodeBytes if the slice contains
+	// additional data after the first RLP value.
+	ErrMoreThanOneValue = errors.New("rlp: input contains more than one value")
+
 	// internal errors
 	errNotInList    = errors.New("rlp: call of ListEnd outside of any list")
 	errNotAtEOL     = errors.New("rlp: call of ListEnd not positioned at EOL")
@@ -611,7 +623,7 @@ func (s *Stream) Bytes() ([]byte, error) {
 		if err = s.readFull(b); err != nil {
 			return nil, err
 		}
-		if size == 1 && b[0] < 56 {
+		if size == 1 && b[0] < 128 {
 			return nil, ErrCanonSize
 		}
 		return b, nil
@@ -675,7 +687,7 @@ func (s *Stream) uint(maxbits int) (uint64, error) {
 			return 0, ErrCanonInt
 		case err != nil:
 			return 0, err
-		case size > 0 && v < 56:
+		case size > 0 && v < 128:
 			return 0, ErrCanonSize
 		default:
 			return v, nil

rlp/decode_test.go

@@ -113,12 +113,16 @@ func TestStreamErrors(t *testing.T) {
 		{"00", calls{"Uint"}, nil, ErrCanonInt},
 		{"820002", calls{"Uint"}, nil, ErrCanonInt},
 		{"8133", calls{"Uint"}, nil, ErrCanonSize},
-		{"8156", calls{"Uint"}, nil, nil},
+		{"817F", calls{"Uint"}, nil, ErrCanonSize},
+		{"8180", calls{"Uint"}, nil, nil},
 
 		// Size tags must use the smallest possible encoding.
 		// Leading zero bytes in the size tag are also rejected.
 		{"8100", calls{"Uint"}, nil, ErrCanonSize},
 		{"8100", calls{"Bytes"}, nil, ErrCanonSize},
+		{"8101", calls{"Bytes"}, nil, ErrCanonSize},
+		{"817F", calls{"Bytes"}, nil, ErrCanonSize},
+		{"8180", calls{"Bytes"}, nil, nil},
 		{"B800", calls{"Kind"}, withoutInputLimit, ErrCanonSize},
 		{"B90000", calls{"Kind"}, withoutInputLimit, ErrCanonSize},
 		{"B90055", calls{"Kind"}, withoutInputLimit, ErrCanonSize},
@@ -132,11 +136,11 @@ func TestStreamErrors(t *testing.T) {
 		{"", calls{"Kind"}, nil, io.EOF},
 		{"", calls{"Uint"}, nil, io.EOF},
 		{"", calls{"List"}, nil, io.EOF},
-		{"8158", calls{"Uint", "Uint"}, nil, io.EOF},
+		{"8180", calls{"Uint", "Uint"}, nil, io.EOF},
 		{"C0", calls{"List", "ListEnd", "List"}, nil, io.EOF},
 
 		{"", calls{"List"}, withoutInputLimit, io.EOF},
-		{"8158", calls{"Uint", "Uint"}, withoutInputLimit, io.EOF},
+		{"8180", calls{"Uint", "Uint"}, withoutInputLimit, io.EOF},
 		{"C0", calls{"List", "ListEnd", "List"}, withoutInputLimit, io.EOF},
 
 		// Input limit errors.
@@ -153,11 +157,11 @@ func TestStreamErrors(t *testing.T) {
 		// down toward zero in Stream.remaining, reading too far can overflow
 		// remaining to a large value, effectively disabling the limit.
 		{"C40102030401", calls{"Raw", "Uint"}, withCustomInputLimit(5), io.EOF},
-		{"C4010203048158", calls{"Raw", "Uint"}, withCustomInputLimit(6), ErrValueTooLarge},
+		{"C4010203048180", calls{"Raw", "Uint"}, withCustomInputLimit(6), ErrValueTooLarge},
 
 		// Check that the same calls are fine without a limit.
 		{"C40102030401", calls{"Raw", "Uint"}, withoutInputLimit, nil},
-		{"C4010203048158", calls{"Raw", "Uint"}, withoutInputLimit, nil},
+		{"C4010203048180", calls{"Raw", "Uint"}, withoutInputLimit, nil},
 
 		// Unexpected EOF. This only happens when there is
 		// no input limit, so the reader needs to be 'dumbed down'.
@@ -349,6 +353,7 @@ var decodeTests = []decodeTest{
 
 	// byte arrays
 	{input: "02", ptr: new([1]byte), value: [1]byte{2}},
+	{input: "8180", ptr: new([1]byte), value: [1]byte{128}},
 	{input: "850102030405", ptr: new([5]byte), value: [5]byte{1, 2, 3, 4, 5}},
 
 	// byte array errors
@@ -359,6 +364,7 @@ var decodeTests = []decodeTest{
 	{input: "C3010203", ptr: new([5]byte), error: "rlp: expected input string or byte for [5]uint8"},
 	{input: "86010203040506", ptr: new([5]byte), error: "rlp: input string too long for [5]uint8"},
 	{input: "8105", ptr: new([1]byte), error: "rlp: non-canonical size information for [1]uint8"},
+	{input: "817F", ptr: new([1]byte), error: "rlp: non-canonical size information for [1]uint8"},
 
 	// zero sized byte arrays
 	{input: "80", ptr: new([0]byte), value: [0]byte{}},
@@ -427,7 +433,8 @@ var decodeTests = []decodeTest{
 	{input: "80", ptr: new(*uint), value: uintp(0)},
 	{input: "C0", ptr: new(*uint), error: "rlp: expected input string or byte for uint"},
 	{input: "07", ptr: new(*uint), value: uintp(7)},
-	{input: "8158", ptr: new(*uint), value: uintp(0x58)},
+	{input: "817F", ptr: new(*uint), error: "rlp: non-canonical size information for uint"},
+	{input: "8180", ptr: new(*uint), value: uintp(0x80)},
 	{input: "C109", ptr: new(*[]uint), value: &[]uint{9}},
 	{input: "C58403030303", ptr: new(*[][]byte), value: &[][]byte{{3, 3, 3, 3}}},
 

tests/init.go

@@ -35,6 +35,7 @@ var (
 	stateTestDir       = filepath.Join(baseDir, "StateTests")
 	transactionTestDir = filepath.Join(baseDir, "TransactionTests")
 	vmTestDir          = filepath.Join(baseDir, "VMTests")
+	rlpTestDir         = filepath.Join(baseDir, "RLPTests")
 
 	BlockSkipTests = []string{
 		// These tests are not valid, as they are out of scope for RLP and

tests/rlp_test.go

@@ -0,0 +1,20 @@
+package tests
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func TestRLP(t *testing.T) {
+	err := RunRLPTest(filepath.Join(rlpTestDir, "rlptest.json"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestRLP_invalid(t *testing.T) {
+	err := RunRLPTest(filepath.Join(rlpTestDir, "invalidRLPTest.json"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

tests/rlp_test_util.go

@@ -0,0 +1,172 @@
+package tests
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"os"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// RLPTest is the JSON structure of a single RLP test.
+type RLPTest struct {
+	// If the value of In is "INVALID" or "VALID", the test
+	// checks whether Out can be decoded into a value of
+	// type interface{}.
+	//
+	// For other JSON values, In is treated as a driver for
+	// calls to rlp.Stream. The test also verifies that encoding
+	// In produces the bytes in Out.
+	In interface{}
+
+	// Out is a hex-encoded RLP value.
+	Out string
+}
+
+// RunRLPTest runs the tests in the given file, skipping tests by name.
+func RunRLPTest(file string, skip []string) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return RunRLPTestWithReader(f, skip)
+}
+
+// RunRLPTest runs the tests encoded in r, skipping tests by name.
+func RunRLPTestWithReader(r io.Reader, skip []string) error {
+	var tests map[string]*RLPTest
+	if err := readJson(r, &tests); err != nil {
+		return err
+	}
+	for _, s := range skip {
+		delete(tests, s)
+	}
+	for name, test := range tests {
+		if err := test.Run(); err != nil {
+			return fmt.Errorf("test %q failed: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// Run executes the test.
+func (t *RLPTest) Run() error {
+	outb, err := hex.DecodeString(t.Out)
+	if err != nil {
+		return fmt.Errorf("invalid hex in Out")
+	}
+
+	// Handle simple decoding tests with no actual In value.
+	if t.In == "VALID" || t.In == "INVALID" {
+		return checkDecodeInterface(outb, t.In == "VALID")
+	}
+
+	// Check whether encoding the value produces the same bytes.
+	in := translateJSON(t.In)
+	b, err := rlp.EncodeToBytes(in)
+	if err != nil {
+		return fmt.Errorf("encode failed: %v", err)
+	}
+	if !bytes.Equal(b, outb) {
+		return fmt.Errorf("encode produced %x, want %x", b, outb)
+	}
+	// Test stream decoding.
+	s := rlp.NewStream(bytes.NewReader(outb), 0)
+	return checkDecodeFromJSON(s, in)
+}
+
+func checkDecodeInterface(b []byte, isValid bool) error {
+	err := rlp.DecodeBytes(b, new(interface{}))
+	switch {
+	case isValid && err != nil:
+		return fmt.Errorf("decoding failed: %v", err)
+	case !isValid && err == nil:
+		return fmt.Errorf("decoding of invalid value succeeded")
+	}
+	return nil
+}
+
+// translateJSON makes test json values encodable with RLP.
+func translateJSON(v interface{}) interface{} {
+	switch v := v.(type) {
+	case float64:
+		return uint64(v)
+	case string:
+		if len(v) > 0 && v[0] == '#' { // # starts a faux big int.
+			big, ok := new(big.Int).SetString(v[1:], 10)
+			if !ok {
+				panic(fmt.Errorf("bad test: bad big int: %q", v))
+			}
+			return big
+		}
+		return []byte(v)
+	case []interface{}:
+		new := make([]interface{}, len(v))
+		for i := range v {
+			new[i] = translateJSON(v[i])
+		}
+		return new
+	default:
+		panic(fmt.Errorf("can't handle %T", v))
+	}
+}
+
+// checkDecodeFromJSON decodes from s guided by exp. exp drives the
+// Stream by invoking decoding operations (Uint, Big, List, ...) based
+// on the type of each value. The value decoded from the RLP stream
+// must match the JSON value.
+func checkDecodeFromJSON(s *rlp.Stream, exp interface{}) error {
+	switch exp := exp.(type) {
+	case uint64:
+		i, err := s.Uint()
+		if err != nil {
+			return addStack("Uint", exp, err)
+		}
+		if i != exp {
+			return addStack("Uint", exp, fmt.Errorf("result mismatch: got %d", i))
+		}
+	case *big.Int:
+		big := new(big.Int)
+		if err := s.Decode(&big); err != nil {
+			return addStack("Big", exp, err)
+		}
+		if big.Cmp(exp) != 0 {
+			return addStack("Big", exp, fmt.Errorf("result mismatch: got %d", big))
+		}
+	case []byte:
+		b, err := s.Bytes()
+		if err != nil {
+			return addStack("Bytes", exp, err)
+		}
+		if !bytes.Equal(b, exp) {
+			return addStack("Bytes", exp, fmt.Errorf("result mismatch: got %x", b))
+		}
+	case []interface{}:
+		if _, err := s.List(); err != nil {
+			return addStack("List", exp, err)
+		}
+		for i, v := range exp {
+			if err := checkDecodeFromJSON(s, v); err != nil {
+				return addStack(fmt.Sprintf("[%d]", i), exp, err)
+			}
+		}
+		if err := s.ListEnd(); err != nil {
+			return addStack("ListEnd", exp, err)
+		}
+	default:
+		panic(fmt.Errorf("unhandled type: %T", exp))
+	}
+	return nil
+}
+
+func addStack(op string, val interface{}, err error) error {
+	lines := strings.Split(err.Error(), "\n")
+	lines = append(lines, fmt.Sprintf("\t%s: %v", op, val))
+	return errors.New(strings.Join(lines, "\n"))
+}