From bb07ce3eedabb2133227ba1d7569d171d8e5b25c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 21 Dec 2015 21:05:20 +0100 Subject: [PATCH] rlp: add "tail" struct tag --- rlp/decode.go | 51 ++++++++++++++++++++++++++-------------- rlp/decode_tail_test.go | 33 ++++++++++++++++++++++++++ rlp/decode_test.go | 52 +++++++++++++++++++++++++++++++++++++++++ rlp/encode.go | 11 +++++---- rlp/encode_test.go | 4 ++++ rlp/typecache.go | 38 ++++++++++++++++++++++++++---- 6 files changed, 163 insertions(+), 26 deletions(-) create mode 100644 rlp/decode_tail_test.go diff --git a/rlp/decode.go b/rlp/decode.go index c4d42c6fc..c4e5869cc 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -63,11 +63,12 @@ type Decoder interface { // must contain an element for each decoded field. Decode returns an // error if there are too few or too many elements. // -// The decoding of struct fields honours one particular struct tag, -// "nil". This tag applies to pointer-typed fields and changes the +// The decoding of struct fields honours two struct tags, "tail" and +// "nil". For an explanation of "tail", see the example. +// The "nil" tag applies to pointer-typed fields and changes the // decoding rules for the field such that input values of size zero -// decode as a nil pointer. This tag can be useful when decoding recursive -// types. +// decode as a nil pointer. This tag can be useful when decoding +// recursive types. // // type StructWithEmptyOK struct { // Foo *[20]byte `rlp:"nil"` @@ -190,7 +191,7 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { case kind == reflect.String: return decodeString, nil case kind == reflect.Slice || kind == reflect.Array: - return makeListDecoder(typ) + return makeListDecoder(typ, tags) case kind == reflect.Struct: return makeStructDecoder(typ) case kind == reflect.Ptr: @@ -264,7 +265,7 @@ func decodeBigInt(s *Stream, val reflect.Value) error { return nil } -func makeListDecoder(typ reflect.Type) (decoder, error) { +func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { etype := typ.Elem() if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { @@ -277,15 +278,26 @@ func makeListDecoder(typ reflect.Type) (decoder, error) { if err != nil { return nil, err } - - isArray := typ.Kind() == reflect.Array - return func(s *Stream, val reflect.Value) error { - if isArray { + var dec decoder + switch { + case typ.Kind() == reflect.Array: + dec = func(s *Stream, val reflect.Value) error { return decodeListArray(s, val, etypeinfo.decoder) - } else { + } + case tag.tail: + // A slice with "tail" tag can occur as the last field + // of a struct and is upposed to swallow all remaining + // list elements. The struct decoder already called s.List, + // proceed directly to decoding the elements. + dec = func(s *Stream, val reflect.Value) error { + return decodeSliceElems(s, val, etypeinfo.decoder) + } + default: + dec = func(s *Stream, val reflect.Value) error { return decodeListSlice(s, val, etypeinfo.decoder) } - }, nil + } + return dec, nil } func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error { @@ -297,7 +309,13 @@ func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error { val.Set(reflect.MakeSlice(val.Type(), 0, 0)) return s.ListEnd() } + if err := decodeSliceElems(s, val, elemdec); err != nil { + return err + } + return s.ListEnd() +} +func decodeSliceElems(s *Stream, val reflect.Value, elemdec decoder) error { i := 0 for ; ; i++ { // grow slice if necessary @@ -323,12 +341,11 @@ func decodeListSlice(s *Stream, val reflect.Value, elemdec decoder) error { if i < val.Len() { val.SetLen(i) } - return s.ListEnd() + return nil } func decodeListArray(s *Stream, val reflect.Value, elemdec decoder) error { - _, err := s.List() - if err != nil { + if _, err := s.List(); err != nil { return wrapStreamError(err, val.Type()) } vlen := val.Len() @@ -398,11 +415,11 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { return nil, err } dec := func(s *Stream, val reflect.Value) (err error) { - if _, err = s.List(); err != nil { + if _, err := s.List(); err != nil { return wrapStreamError(err, typ) } for _, f := range fields { - err = f.info.decoder(s, val.Field(f.index)) + err := f.info.decoder(s, val.Field(f.index)) if err == EOL { return &decodeError{msg: "too few elements", typ: typ} } else if err != nil { diff --git a/rlp/decode_tail_test.go b/rlp/decode_tail_test.go new file mode 100644 index 000000000..885354390 --- /dev/null +++ b/rlp/decode_tail_test.go @@ -0,0 +1,33 @@ +package rlp + +import ( + "bytes" + "fmt" +) + +type structWithTail struct { + A, B uint + C []uint `rlp:"tail"` +} + +func ExampleDecode_structTagTail() { + // In this example, the "tail" struct tag is used to decode lists of + // differing length into a struct. + var val structWithTail + + err := Decode(bytes.NewReader([]byte{0xC4, 0x01, 0x02, 0x03, 0x04}), &val) + fmt.Printf("with 4 elements: err=%v val=%v\n", err, val) + + err = Decode(bytes.NewReader([]byte{0xC6, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), &val) + fmt.Printf("with 6 elements: err=%v val=%v\n", err, val) + + // Note that at least two list elements must be present to + // fill fields A and B: + err = Decode(bytes.NewReader([]byte{0xC1, 0x01}), &val) + fmt.Printf("with 1 element: err=%q\n", err) + + // Output: + // with 4 elements: err= val={1 2 [3 4]} + // with 6 elements: err= val={1 2 [3 4 5 6]} + // with 1 element: err="rlp: too few elements for rlp.structWithTail" +} diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 408f1a5a9..2d465b74d 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -312,6 +312,26 @@ type recstruct struct { Child *recstruct `rlp:"nil"` } +type invalidTail1 struct { + A uint `rlp:"tail"` + B string +} + +type invalidTail2 struct { + A uint + B string `rlp:"tail"` +} + +type tailRaw struct { + A uint + Tail []RawValue `rlp:"tail"` +} + +type tailUint struct { + A uint + Tail []uint `rlp:"tail"` +} + var ( veryBigInt = big.NewInt(0).Add( big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), @@ -437,6 +457,38 @@ var decodeTests = []decodeTest{ ptr: new(recstruct), error: "rlp: expected input string or byte for uint, decoding into (rlp.recstruct).Child.I", }, + { + input: "C0", + ptr: new(invalidTail1), + error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail1.A (must be on last field)", + }, + { + input: "C0", + ptr: new(invalidTail2), + error: "rlp: invalid struct tag \"tail\" for rlp.invalidTail2.B (field type is not slice)", + }, + { + input: "C50102C20102", + ptr: new(tailUint), + error: "rlp: expected input string or byte for uint, decoding into (rlp.tailUint).Tail[1]", + }, + + // struct tag "tail" + { + input: "C3010203", + ptr: new(tailRaw), + value: tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, + }, + { + input: "C20102", + ptr: new(tailRaw), + value: tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, + }, + { + input: "C101", + ptr: new(tailRaw), + value: tailRaw{A: 1, Tail: []RawValue{}}, + }, // RawValue {input: "01", ptr: new(RawValue), value: RawValue(unhex("01"))}, diff --git a/rlp/encode.go b/rlp/encode.go index d73b17c28..17cfc6b66 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -345,7 +345,7 @@ var ( ) // makeWriter creates a writer function for the given type. -func makeWriter(typ reflect.Type) (writer, error) { +func makeWriter(typ reflect.Type, ts tags) (writer, error) { kind := typ.Kind() switch { case typ == rawValueType: @@ -371,7 +371,7 @@ func makeWriter(typ reflect.Type) (writer, error) { case kind == reflect.Array && isByte(typ.Elem()): return writeByteArray, nil case kind == reflect.Slice || kind == reflect.Array: - return makeSliceWriter(typ) + return makeSliceWriter(typ, ts) case kind == reflect.Struct: return makeStructWriter(typ) case kind == reflect.Ptr: @@ -507,20 +507,21 @@ func writeInterface(val reflect.Value, w *encbuf) error { return ti.writer(eval, w) } -func makeSliceWriter(typ reflect.Type) (writer, error) { +func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { etypeinfo, err := cachedTypeInfo1(typ.Elem(), tags{}) if err != nil { return nil, err } writer := func(val reflect.Value, w *encbuf) error { - lh := w.list() + if !ts.tail { + defer w.listEnd(w.list()) + } vlen := val.Len() for i := 0; i < vlen; i++ { if err := etypeinfo.writer(val.Index(i), w); err != nil { return err } } - w.listEnd(lh) return nil } return writer, nil diff --git a/rlp/encode_test.go b/rlp/encode_test.go index a3f30d804..6f38294e4 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -214,6 +214,10 @@ var encTests = []encTest{ {val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"}, {val: &recstruct{5, nil}, output: "C205C0"}, {val: &recstruct{5, &recstruct{4, &recstruct{3, nil}}}, output: "C605C404C203C0"}, + {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02"), unhex("03")}}, output: "C3010203"}, + {val: &tailRaw{A: 1, Tail: []RawValue{unhex("02")}}, output: "C20102"}, + {val: &tailRaw{A: 1, Tail: []RawValue{}}, output: "C101"}, + {val: &tailRaw{A: 1, Tail: nil}, output: "C101"}, // nil {val: (*uint)(nil), output: "80"}, diff --git a/rlp/typecache.go b/rlp/typecache.go index 0ab096695..a2f217c66 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -17,7 +17,9 @@ package rlp import ( + "fmt" "reflect" + "strings" "sync" ) @@ -33,7 +35,13 @@ type typeinfo struct { // represents struct tags type tags struct { + // rlp:"nil" controls whether empty input results in a nil pointer. nilOK bool + + // rlp:"tail" controls whether this field swallows additional list + // elements. It can only be set for the last field, which must be + // of slice type. + tail bool } type typekey struct { @@ -89,7 +97,10 @@ type field struct { func structFields(typ reflect.Type) (fields []field, err error) { for i := 0; i < typ.NumField(); i++ { if f := typ.Field(i); f.PkgPath == "" { // exported - tags := parseStructTag(f.Tag.Get("rlp")) + tags, err := parseStructTag(typ, i) + if err != nil { + return nil, err + } info, err := cachedTypeInfo1(f.Type, tags) if err != nil { return nil, err @@ -100,8 +111,27 @@ func structFields(typ reflect.Type) (fields []field, err error) { return fields, nil } -func parseStructTag(tag string) tags { - return tags{nilOK: tag == "nil"} +func parseStructTag(typ reflect.Type, fi int) (tags, error) { + f := typ.Field(fi) + var ts tags + for _, t := range strings.Split(f.Tag.Get("rlp"), ",") { + switch t = strings.TrimSpace(t); t { + case "": + case "nil": + ts.nilOK = true + case "tail": + ts.tail = true + if fi != typ.NumField()-1 { + return ts, fmt.Errorf(`rlp: invalid struct tag "tail" for %v.%s (must be on last field)`, typ, f.Name) + } + if f.Type.Kind() != reflect.Slice { + return ts, fmt.Errorf(`rlp: invalid struct tag "tail" for %v.%s (field type is not slice)`, typ, f.Name) + } + default: + return ts, fmt.Errorf("rlp: unknown struct tag %q on %v.%s", t, typ, f.Name) + } + } + return ts, nil } func genTypeInfo(typ reflect.Type, tags tags) (info *typeinfo, err error) { @@ -109,7 +139,7 @@ func genTypeInfo(typ reflect.Type, tags tags) (info *typeinfo, err error) { if info.decoder, err = makeDecoder(typ, tags); err != nil { return nil, err } - if info.writer, err = makeWriter(typ); err != nil { + if info.writer, err = makeWriter(typ, tags); err != nil { return nil, err } return info, nil