LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

fix(orm)!: timestamp encoding doesn't handle nil values properly (#12273)

Browse Source 
Co-authored-by: Julien Robert <julien@rbrt.fr>
This commit is contained in:
Aaron Craelius
2023-02-09 12:25:04 -05:00
committed by GitHub
co-authored by Julien Robert
parent ad847c4e4d
commit dc20731bdd
16 changed files with 418 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orm/encoding/encodeutil/util.go
+6 -1
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/binary"
"io"
"reflect"
"google.golang.org/protobuf/reflect/protoreflect"
)
@@ -42,7 +43,11 @@ func ValuesOf(values ...interface{}) []protoreflect.Value {
value := values[i]
switch value.(type) {
case protoreflect.ProtoMessage:
value = value.(protoreflect.ProtoMessage).ProtoReflect()
if !reflect.ValueOf(value).IsNil() {
value = value.(protoreflect.ProtoMessage).ProtoReflect()
} else {
value = nil
}
}
res[i] = protoreflect.ValueOf(value)
}
orm/encoding/ormfield/bool.go
+10 -5
View File
@@ -21,17 +21,22 @@ var (
func (b BoolCodec) Encode(value protoreflect.Value, w io.Writer) error {
var err error
if value.Bool() {
_, err = w.Write(oneBz)
} else {
if !value.IsValid() || !value.Bool() {
_, err = w.Write(zeroBz)
} else {
_, err = w.Write(oneBz)
}
return err
}
func (b BoolCodec) Compare(v1, v2 protoreflect.Value) int {
b1 := v1.Bool()
b2 := v2.Bool()
var b1, b2 bool
if v1.IsValid() {
b1 = v1.Bool()
}
if v2.IsValid() {
b2 = v2.Bool()
}
if b1 == b2 {
return 0
} else if b1 {
orm/encoding/ormfield/bytes.go
+23 -3
View File
@@ -22,6 +22,9 @@ func (b BytesCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
}
func bytesSize(value protoreflect.Value) int {
if !value.IsValid() {
return 0
}
return len(value.Bytes())
}
@@ -35,12 +38,15 @@ func (b BytesCodec) Decode(r Reader) (protoreflect.Value, error) {
}
func (b BytesCodec) Encode(value protoreflect.Value, w io.Writer) error {
if !value.IsValid() {
return nil
}
_, err := w.Write(value.Bytes())
return err
}
func (b BytesCodec) Compare(v1, v2 protoreflect.Value) int {
return bytes.Compare(v1.Bytes(), v2.Bytes())
return compareBytes(v1, v2)
}
// NonTerminalBytesCodec encodes bytes as raw bytes length prefixed by a single
@@ -69,7 +75,7 @@ func (b NonTerminalBytesCodec) IsOrdered() bool {
}
func (b NonTerminalBytesCodec) Compare(v1, v2 protoreflect.Value) int {
return bytes.Compare(v1.Bytes(), v2.Bytes())
return compareBytes(v1, v2)
}
func (b NonTerminalBytesCodec) Decode(r Reader) (protoreflect.Value, error) {
@@ -88,7 +94,10 @@ func (b NonTerminalBytesCodec) Decode(r Reader) (protoreflect.Value, error) {
}
func (b NonTerminalBytesCodec) Encode(value protoreflect.Value, w io.Writer) error {
bz := value.Bytes()
var bz []byte
if value.IsValid() {
bz = value.Bytes()
}
n := len(bz)
var prefix [binary.MaxVarintLen64]byte
prefixLen := binary.PutUvarint(prefix[:], uint64(n))
@@ -99,3 +108,14 @@ func (b NonTerminalBytesCodec) Encode(value protoreflect.Value, w io.Writer) err
_, err = w.Write(bz)
return err
}
func compareBytes(v1, v2 protoreflect.Value) int {
var bz1, bz2 []byte
if v1.IsValid() {
bz1 = v1.Bytes()
}
if v2.IsValid() {
bz2 = v2.Bytes()
}
return bytes.Compare(bz1, bz2)
}
orm/encoding/ormfield/codec_test.go
+35
View File
@@ -4,6 +4,9 @@ import (
"bytes"
"fmt"
"testing"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/cosmos/cosmos-sdk/orm/encoding/ormfield"
@@ -169,3 +172,35 @@ func TestCompactUInt64(t *testing.T) {
assert.Equal(t, y, y2)
})
}
func TestTimestamp(t *testing.T) {
cdc := ormfield.TimestampCodec{}
// nil value
buf := &bytes.Buffer{}
assert.NilError(t, cdc.Encode(protoreflect.Value{}, buf))
assert.Equal(t, 1, len(buf.Bytes()))
val, err := cdc.Decode(buf)
assert.NilError(t, err)
assert.Assert(t, !val.IsValid())
// no nanos
ts := timestamppb.New(time.Date(2022, 1, 1, 12, 30, 15, 0, time.UTC))
val = protoreflect.ValueOfMessage(ts.ProtoReflect())
buf = &bytes.Buffer{}
assert.NilError(t, cdc.Encode(val, buf))
assert.Equal(t, 6, len(buf.Bytes()))
val2, err := cdc.Decode(buf)
assert.NilError(t, err)
assert.Equal(t, 0, cdc.Compare(val, val2))
// nanos
ts = timestamppb.New(time.Date(2022, 1, 1, 12, 30, 15, 235809753, time.UTC))
val = protoreflect.ValueOfMessage(ts.ProtoReflect())
buf = &bytes.Buffer{}
assert.NilError(t, cdc.Encode(val, buf))
assert.Equal(t, 9, len(buf.Bytes()))
val2, err = cdc.Decode(buf)
assert.NilError(t, err)
assert.Equal(t, 0, cdc.Compare(val, val2))
}
orm/encoding/ormfield/enum.go
+11 -3
View File
@@ -16,7 +16,10 @@ func (e EnumCodec) Decode(r Reader) (protoreflect.Value, error) {
}
func (e EnumCodec) Encode(value protoreflect.Value, w io.Writer) error {
x := value.Enum()
var x protoreflect.EnumNumber
if value.IsValid() {
x = value.Enum()
}
buf := make([]byte, binary.MaxVarintLen32)
n := binary.PutVarint(buf, int64(x))
_, err := w.Write(buf[:n])
@@ -24,8 +27,13 @@ func (e EnumCodec) Encode(value protoreflect.Value, w io.Writer) error {
}
func (e EnumCodec) Compare(v1, v2 protoreflect.Value) int {
x := v1.Enum()
y := v2.Enum()
var x, y protoreflect.EnumNumber
if v1.IsValid() {
x = v1.Enum()
}
if v2.IsValid() {
y = v2.Enum()
}
if x == y {
return 0
} else if x < y {
orm/encoding/ormfield/int32.go
+4 -1
View File
@@ -27,7 +27,10 @@ func (i Int32Codec) Decode(r Reader) (protoreflect.Value, error) {
}
func (i Int32Codec) Encode(value protoreflect.Value, w io.Writer) error {
x := value.Int()
var x int64
if value.IsValid() {
x = value.Int()
}
x += int32Offset
return binary.Write(w, binary.BigEndian, uint32(x))
}
orm/encoding/ormfield/int64.go
+11 -3
View File
@@ -29,7 +29,10 @@ func (i Int64Codec) Decode(r Reader) (protoreflect.Value, error) {
}
func (i Int64Codec) Encode(value protoreflect.Value, w io.Writer) error {
x := value.Int()
var x int64
if value.IsValid() {
x = value.Int()
}
if x >= -1 {
y := uint64(x) + int64Max + 1
return binary.Write(w, binary.BigEndian, y)
@@ -57,8 +60,13 @@ func (i Int64Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
}
func compareInt(v1, v2 protoreflect.Value) int {
x := v1.Int()
y := v2.Int()
var x, y int64
if v1.IsValid() {
x = v1.Int()
}
if v2.IsValid() {
y = v2.Int()
}
if x == y {
return 0
} else if x < y {
orm/encoding/ormfield/string.go
+26 -4
View File
@@ -16,6 +16,10 @@ func (s StringCodec) FixedBufferSize() int {
}
func (s StringCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
if !value.IsValid() {
return 0, nil
}
return len(value.String()), nil
}
@@ -24,7 +28,7 @@ func (s StringCodec) IsOrdered() bool {
}
func (s StringCodec) Compare(v1, v2 protoreflect.Value) int {
return strings.Compare(v1.String(), v2.String())
return compareStrings(v1, v2)
}
func (s StringCodec) Decode(r Reader) (protoreflect.Value, error) {
@@ -33,7 +37,11 @@ func (s StringCodec) Decode(r Reader) (protoreflect.Value, error) {
}
func (s StringCodec) Encode(value protoreflect.Value, w io.Writer) error {
_, err := w.Write([]byte(value.String()))
var x string
if value.IsValid() {
x = value.String()
}
_, err := w.Write([]byte(x))
return err
}
@@ -54,7 +62,7 @@ func (s NonTerminalStringCodec) IsOrdered() bool {
}
func (s NonTerminalStringCodec) Compare(v1, v2 protoreflect.Value) int {
return strings.Compare(v1.String(), v2.String())
return compareStrings(v1, v2)
}
func (s NonTerminalStringCodec) Decode(r Reader) (protoreflect.Value, error) {
@@ -69,7 +77,10 @@ func (s NonTerminalStringCodec) Decode(r Reader) (protoreflect.Value, error) {
}
func (s NonTerminalStringCodec) Encode(value protoreflect.Value, w io.Writer) error {
str := value.String()
var str string
if value.IsValid() {
str = value.String()
}
bz := []byte(str)
for _, b := range bz {
if b == 0 {
@@ -85,3 +96,14 @@ func (s NonTerminalStringCodec) Encode(value protoreflect.Value, w io.Writer) er
}
var nullTerminator = []byte{0}
func compareStrings(v1, v2 protoreflect.Value) int {
var x, y string
if v1.IsValid() {
x = v1.String()
}
if v2.IsValid() {
y = v2.String()
}
return strings.Compare(x, y)
}
orm/encoding/ormfield/timestamp.go
+192 -28
View File
@@ -1,51 +1,151 @@
package ormfield
import (
"fmt"
"io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// TimestampCodec DurationCodec encodes a google.protobuf.Timestamp value as 12 bytes using
// Int64Codec for seconds followed by Int32Codec for nanos. This allows for
// sorted iteration.
// TimestampCodec encodes google.protobuf.Timestamp values with the following
// encoding:
// - nil is encoded as []byte{0xFF}
// - seconds (which can range from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z) is encoded as 5 fixed bytes
// - nanos (which can range from 0 to 999,999,999) is encoded as:
// - []byte{0x0} for zero nanos
// - 4 fixed bytes with the bit mask 0xC0 applied to the first byte
//
// When iterating over timestamp indexes, nil values will always be ordered last.
//
// Values for seconds and nanos outside the ranges specified by google.protobuf.Timestamp will be rejected.
type TimestampCodec struct{}
var (
timestampSecondsField = timestampMsgType.Descriptor().Fields().ByName("seconds")
timestampNanosField = timestampMsgType.Descriptor().Fields().ByName("nanos")
const (
timestampNilValue = 0xFF
timestampZeroNanosValue = 0x0
timestampSecondsMin = -62135579038
timestampSecondsMax = 253402318799
timestampNanosMax = 999999999
)
func getTimestampSecondsAndNanos(value protoreflect.Value) (protoreflect.Value, protoreflect.Value) {
msg := value.Message()
return msg.Get(timestampSecondsField), msg.Get(timestampNanosField)
}
func (t TimestampCodec) Decode(r Reader) (protoreflect.Value, error) {
seconds, err := int64Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
nanos, err := int32Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
msg := timestampMsgType.New()
msg.Set(timestampSecondsField, seconds)
msg.Set(timestampNanosField, nanos)
return protoreflect.ValueOfMessage(msg), nil
}
var (
timestampNilBz = []byte{timestampNilValue}
timestampZeroNanosBz = []byte{timestampZeroNanosValue}
)
func (t TimestampCodec) Encode(value protoreflect.Value, w io.Writer) error {
// nil case
if !value.IsValid() {
_, err := w.Write(timestampNilBz)
return err
}
seconds, nanos := getTimestampSecondsAndNanos(value)
err := int64Codec.Encode(seconds, w)
secondsInt := seconds.Int()
if secondsInt < timestampSecondsMin || secondsInt > timestampSecondsMax {
return fmt.Errorf("seconds is out of range %d, must be between %d and %d", secondsInt, timestampSecondsMin, timestampSecondsMax)
}
secondsInt -= timestampSecondsMin
var secondsBz [5]byte
// write the seconds buffer from the end to the front
for i := 4; i >= 0; i-- {
secondsBz[i] = byte(secondsInt)
secondsInt >>= 8
}
_, err := w.Write(secondsBz[:])
if err != nil {
return err
}
return int32Codec.Encode(nanos, w)
nanosInt := nanos.Int()
if nanosInt == 0 {
_, err = w.Write(timestampZeroNanosBz)
return err
}
if nanosInt < 0 || nanosInt > timestampNanosMax {
return fmt.Errorf("nanos is out of range %d, must be between %d and %d", secondsInt, 0, timestampNanosMax)
}
var nanosBz [4]byte
for i := 3; i >= 0; i-- {
nanosBz[i] = byte(nanosInt)
nanosInt >>= 8
}
nanosBz[0] = nanosBz[0] | 0xC0
_, err = w.Write(nanosBz[:])
return err
}
func (t TimestampCodec) Decode(r Reader) (protoreflect.Value, error) {
b0, err := r.ReadByte()
if err != nil {
return protoreflect.Value{}, err
}
if b0 == timestampNilValue {
return protoreflect.Value{}, nil
}
var secondsBz [4]byte
n, err := r.Read(secondsBz[:])
if err != nil {
return protoreflect.Value{}, err
}
if n < 4 {
return protoreflect.Value{}, io.EOF
}
var seconds = int64(b0)
for i := 0; i < 4; i++ {
seconds <<= 8
seconds |= int64(secondsBz[i])
}
seconds += timestampSecondsMin
msg := timestampMsgType.New()
msg.Set(timestampSecondsField, protoreflect.ValueOfInt64(seconds))
b0, err = r.ReadByte()
if err != nil {
return protoreflect.Value{}, err
}
if b0 == timestampZeroNanosValue {
return protoreflect.ValueOfMessage(msg), nil
}
var nanosBz [3]byte
n, err = r.Read(nanosBz[:])
if err != nil {
return protoreflect.Value{}, err
}
if n < 3 {
return protoreflect.Value{}, io.EOF
}
var nanos = int32(b0) & 0x3F // clear first two bits
for i := 0; i < 3; i++ {
nanos <<= 8
nanos |= int32(nanosBz[i])
}
msg.Set(timestampNanosField, protoreflect.ValueOfInt32(nanos))
return protoreflect.ValueOfMessage(msg), nil
}
func (t TimestampCodec) Compare(v1, v2 protoreflect.Value) int {
if !v1.IsValid() {
if !v2.IsValid() {
return 0
}
return 1
}
if !v2.IsValid() {
return -1
}
s1, n1 := getTimestampSecondsAndNanos(v1)
s2, n2 := getTimestampSecondsAndNanos(v2)
c := compareInt(s1, s2)
@@ -61,9 +161,73 @@ func (t TimestampCodec) IsOrdered() bool {
}
func (t TimestampCodec) FixedBufferSize() int {
return 12
return 9
}
func (t TimestampCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
return 9, nil
}
// TimestampV0Codec encodes a google.protobuf.Timestamp value as 12 bytes using
// Int64Codec for seconds followed by Int32Codec for nanos. This type does not
// encode nil values correctly, but is retained in order to allow users of the
// previous encoding to successfully migrate from this encoding to the new encoding
// specified by TimestampCodec.
type TimestampV0Codec struct{}
var (
timestampSecondsField = timestampMsgType.Descriptor().Fields().ByName("seconds")
timestampNanosField = timestampMsgType.Descriptor().Fields().ByName("nanos")
)
func getTimestampSecondsAndNanos(value protoreflect.Value) (protoreflect.Value, protoreflect.Value) {
msg := value.Message()
return msg.Get(timestampSecondsField), msg.Get(timestampNanosField)
}
func (t TimestampV0Codec) Decode(r Reader) (protoreflect.Value, error) {
seconds, err := int64Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
nanos, err := int32Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
msg := timestampMsgType.New()
msg.Set(timestampSecondsField, seconds)
msg.Set(timestampNanosField, nanos)
return protoreflect.ValueOfMessage(msg), nil
}
func (t TimestampV0Codec) Encode(value protoreflect.Value, w io.Writer) error {
seconds, nanos := getTimestampSecondsAndNanos(value)
err := int64Codec.Encode(seconds, w)
if err != nil {
return err
}
return int32Codec.Encode(nanos, w)
}
func (t TimestampV0Codec) Compare(v1, v2 protoreflect.Value) int {
s1, n1 := getTimestampSecondsAndNanos(v1)
s2, n2 := getTimestampSecondsAndNanos(v2)
c := compareInt(s1, s2)
if c != 0 {
return c
} else {
return compareInt(n1, n2)
}
}
func (t TimestampV0Codec) IsOrdered() bool {
return true
}
func (t TimestampV0Codec) FixedBufferSize() int {
return 12
}
func (t TimestampV0Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
return t.FixedBufferSize(), nil
}
orm/encoding/ormfield/uint32.go
+10 -2
View File
@@ -34,7 +34,11 @@ func (u FixedUint32Codec) Decode(r Reader) (protoreflect.Value, error) {
}
func (u FixedUint32Codec) Encode(value protoreflect.Value, w io.Writer) error {
return binary.Write(w, binary.BigEndian, uint32(value.Uint()))
var x uint64
if value.IsValid() {
x = value.Uint()
}
return binary.Write(w, binary.BigEndian, uint32(x))
}
// CompactUint32Codec encodes uint32 values using EncodeCompactUint32.
@@ -46,7 +50,11 @@ func (c CompactUint32Codec) Decode(r Reader) (protoreflect.Value, error) {
}
func (c CompactUint32Codec) Encode(value protoreflect.Value, w io.Writer) error {
_, err := w.Write(EncodeCompactUint32(uint32(value.Uint())))
var x uint64
if value.IsValid() {
x = value.Uint()
}
_, err := w.Write(EncodeCompactUint32(uint32(x)))
return err
}
orm/encoding/ormfield/uint64.go
+17 -4
View File
@@ -34,12 +34,21 @@ func (u FixedUint64Codec) Decode(r Reader) (protoreflect.Value, error) {
}
func (u FixedUint64Codec) Encode(value protoreflect.Value, w io.Writer) error {
return binary.Write(w, binary.BigEndian, value.Uint())
var x uint64
if value.IsValid() {
x = value.Uint()
}
return binary.Write(w, binary.BigEndian, x)
}
func compareUint(v1, v2 protoreflect.Value) int {
x := v1.Uint()
y := v2.Uint()
var x, y uint64
if v1.IsValid() {
x = v1.Uint()
}
if v2.IsValid() {
y = v2.Uint()
}
if x == y {
return 0
} else if x < y {
@@ -58,7 +67,11 @@ func (c CompactUint64Codec) Decode(r Reader) (protoreflect.Value, error) {
}
func (c CompactUint64Codec) Encode(value protoreflect.Value, w io.Writer) error {
_, err := w.Write(EncodeCompactUint64(value.Uint()))
var x uint64
if value.IsValid() {
x = value.Uint()
}
_, err := w.Write(EncodeCompactUint64(x))
return err
}
orm/encoding/ormkv/key_codec.go
+7 -2
View File
@@ -104,7 +104,9 @@ func (cdc *KeyCodec) EncodeKey(values []protoreflect.Value) ([]byte, error) {
func (cdc *KeyCodec) GetKeyValues(message protoreflect.Message) []protoreflect.Value {
res := make([]protoreflect.Value, len(cdc.fieldDescriptors))
for i, f := range cdc.fieldDescriptors {
res[i] = message.Get(f)
if message.Has(f) {
res[i] = message.Get(f)
}
}
return res
}
@@ -209,7 +211,10 @@ func (cdc KeyCodec) ComputeKeyBufferSize(values []protoreflect.Value) (int, erro
// supported.
func (cdc *KeyCodec) SetKeyValues(message protoreflect.Message, values []protoreflect.Value) {
for i, f := range cdc.fieldDescriptors {
message.Set(f, values[i])
value := values[i]
if value.IsValid() {
message.Set(f, value)
}
}
}