feat(orm): add ormfield.Codec (#10601)

* feat(orm): add ormvalue.Codec

* WIP

* WIP

* working tests

* update dep

* support more types, add docs

* comments

* address review comments

* updates

* add comment
This commit is contained in:
Aaron Craelius
2021-11-24 13:10:26 -05:00
committed by GitHub
parent 60483cd2bf
commit 6662f2ff30
25 changed files with 5599 additions and 10 deletions
+54
View File
@@ -0,0 +1,54 @@
package ormfield
import (
io "io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// BoolCodec encodes a bool value as a single byte 0 or 1.
type BoolCodec struct{}
func (b BoolCodec) Decode(r Reader) (protoreflect.Value, error) {
x, err := r.ReadByte()
return protoreflect.ValueOfBool(x != 0), err
}
var (
zeroBz = []byte{0}
oneBz = []byte{1}
)
func (b BoolCodec) Encode(value protoreflect.Value, w io.Writer) error {
var err error
if value.Bool() {
_, err = w.Write(oneBz)
} else {
_, err = w.Write(zeroBz)
}
return err
}
func (b BoolCodec) Compare(v1, v2 protoreflect.Value) int {
b1 := v1.Bool()
b2 := v2.Bool()
if b1 == b2 {
return 0
} else if b1 {
return -1
} else {
return 1
}
}
func (b BoolCodec) IsOrdered() bool {
return false
}
func (b BoolCodec) FixedBufferSize() int {
return 1
}
func (b BoolCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
return b.FixedBufferSize(), nil
}
+99
View File
@@ -0,0 +1,99 @@
package ormfield
import (
"bytes"
"io"
"google.golang.org/protobuf/reflect/protoreflect"
"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
)
// BytesCodec encodes bytes as raw bytes. It errors if the byte array is longer
// than 255 bytes.
type BytesCodec struct{}
func (b BytesCodec) FixedBufferSize() int {
return -1
}
func (b BytesCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
return bytesSize(value)
}
func bytesSize(value protoreflect.Value) (int, error) {
bz := value.Bytes()
n := len(bz)
if n > 255 {
return -1, ormerrors.BytesFieldTooLong
}
return n, nil
}
func (b BytesCodec) IsOrdered() bool {
return false
}
func (b BytesCodec) Decode(r Reader) (protoreflect.Value, error) {
bz, err := io.ReadAll(r)
return protoreflect.ValueOfBytes(bz), err
}
func (b BytesCodec) Encode(value protoreflect.Value, w io.Writer) error {
_, err := w.Write(value.Bytes())
return err
}
func (b BytesCodec) Compare(v1, v2 protoreflect.Value) int {
return bytes.Compare(v1.Bytes(), v2.Bytes())
}
// NonTerminalBytesCodec encodes bytes as raw bytes length prefixed by a single
// byte. It errors if the byte array is longer than 255 bytes.
type NonTerminalBytesCodec struct{}
func (b NonTerminalBytesCodec) FixedBufferSize() int {
return -1
}
func (b NonTerminalBytesCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
n, err := bytesSize(value)
return n + 1, err
}
func (b NonTerminalBytesCodec) IsOrdered() bool {
return false
}
func (b NonTerminalBytesCodec) Compare(v1, v2 protoreflect.Value) int {
return bytes.Compare(v1.Bytes(), v2.Bytes())
}
func (b NonTerminalBytesCodec) Decode(r Reader) (protoreflect.Value, error) {
n, err := r.ReadByte()
if err != nil {
return protoreflect.Value{}, err
}
if n == 0 {
return protoreflect.ValueOfBytes([]byte{}), nil
}
bz := make([]byte, n)
_, err = r.Read(bz)
return protoreflect.ValueOfBytes(bz), err
}
func (b NonTerminalBytesCodec) Encode(value protoreflect.Value, w io.Writer) error {
bz := value.Bytes()
n := len(bz)
if n > 255 {
return ormerrors.BytesFieldTooLong
}
_, err := w.Write([]byte{byte(n)})
if err != nil {
return err
}
_, err = w.Write(bz)
return err
}
+106
View File
@@ -0,0 +1,106 @@
package ormfield
import (
"io"
"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"
"google.golang.org/protobuf/reflect/protoreflect"
)
// Codec defines an interface for decoding and encoding values in ORM index keys.
type Codec interface {
// Decode decodes a value in a key.
Decode(r Reader) (protoreflect.Value, error)
// Encode encodes a value in a key.
Encode(value protoreflect.Value, w io.Writer) error
// Compare compares two values of this type and should primarily be used
// for testing.
Compare(v1, v2 protoreflect.Value) int
// IsOrdered returns true if callers can always assume that this ordering
// is suitable for sorted iteration.
IsOrdered() bool
// FixedBufferSize returns a positive value if encoders should assume a
// fixed size buffer for encoding. Encoders will use at most this much size
// to encode the value.
FixedBufferSize() int
// ComputeBufferSize estimates the buffer size needed to encode the field.
// Encoders will use at most this much size to encode the value.
ComputeBufferSize(value protoreflect.Value) (int, error)
}
type Reader interface {
io.Reader
io.ByteReader
}
var (
timestampMsgType = (&timestamppb.Timestamp{}).ProtoReflect().Type()
timestampFullName = timestampMsgType.Descriptor().FullName()
durationMsgType = (&durationpb.Duration{}).ProtoReflect().Type()
durationFullName = durationMsgType.Descriptor().FullName()
)
// GetCodec returns the Codec for the provided field if one is defined.
// nonTerminal should be set to true if this value is being encoded as a
// non-terminal segment of a multi-part key.
func GetCodec(field protoreflect.FieldDescriptor, nonTerminal bool) (Codec, error) {
if field == nil {
return nil, ormerrors.UnsupportedKeyField.Wrap("nil field")
}
if field.IsList() {
return nil, ormerrors.UnsupportedKeyField.Wrapf("repeated field %s", field.FullName())
}
if field.ContainingOneof() != nil {
return nil, ormerrors.UnsupportedKeyField.Wrapf("oneof field %s", field.FullName())
}
switch field.Kind() {
case protoreflect.BytesKind:
if nonTerminal {
return NonTerminalBytesCodec{}, nil
} else {
return BytesCodec{}, nil
}
case protoreflect.StringKind:
if nonTerminal {
return NonTerminalStringCodec{}, nil
} else {
return StringCodec{}, nil
}
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
return Uint32Codec{}, nil
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
return Uint64Codec{}, nil
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
return Int32Codec{}, nil
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
return Int64Codec{}, nil
case protoreflect.BoolKind:
return BoolCodec{}, nil
case protoreflect.EnumKind:
return EnumCodec{}, nil
case protoreflect.MessageKind:
msgName := field.Message().FullName()
switch msgName {
case timestampFullName:
return TimestampCodec{}, nil
case durationFullName:
return DurationCodec{}, nil
default:
return nil, ormerrors.UnsupportedKeyField.Wrapf("%s of type %s", field.FullName(), msgName)
}
default:
return nil, ormerrors.UnsupportedKeyField.Wrapf("%s of kind %s", field.FullName(), field.Kind())
}
}
+87
View File
@@ -0,0 +1,87 @@
package ormfield_test
import (
"bytes"
"fmt"
"testing"
"github.com/cosmos/cosmos-sdk/orm/encoding/ormfield"
"google.golang.org/protobuf/reflect/protoreflect"
"gotest.tools/v3/assert"
"pgregory.net/rapid"
"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
"github.com/cosmos/cosmos-sdk/orm/internal/testutil"
)
func TestCodec(t *testing.T) {
for _, ks := range testutil.TestFieldSpecs {
testCodec(t, ks)
}
}
func testCodec(t *testing.T, spec testutil.TestFieldSpec) {
t.Run(fmt.Sprintf("%s %v", spec.FieldName, false), func(t *testing.T) {
testCodecNT(t, spec.FieldName, spec.Gen, false)
})
t.Run(fmt.Sprintf("%s %v", spec.FieldName, true), func(t *testing.T) {
testCodecNT(t, spec.FieldName, spec.Gen, true)
})
}
func testCodecNT(t *testing.T, fname protoreflect.Name, generator *rapid.Generator, nonTerminal bool) {
cdc, err := testutil.MakeTestCodec(fname, nonTerminal)
assert.NilError(t, err)
rapid.Check(t, func(t *rapid.T) {
x := protoreflect.ValueOf(generator.Draw(t, string(fname)))
bz1 := checkEncodeDecodeSize(t, x, cdc)
if cdc.IsOrdered() {
y := protoreflect.ValueOf(generator.Draw(t, fmt.Sprintf("%s 2", fname)))
bz2 := checkEncodeDecodeSize(t, y, cdc)
assert.Equal(t, cdc.Compare(x, y), bytes.Compare(bz1, bz2))
}
})
}
func checkEncodeDecodeSize(t *rapid.T, x protoreflect.Value, cdc ormfield.Codec) []byte {
buf := &bytes.Buffer{}
err := cdc.Encode(x, buf)
assert.NilError(t, err)
bz := buf.Bytes()
size, err := cdc.ComputeBufferSize(x)
assert.NilError(t, err)
assert.Assert(t, size >= len(bz))
fixedSize := cdc.FixedBufferSize()
if fixedSize > 0 {
assert.Equal(t, fixedSize, size)
}
y, err := cdc.Decode(bytes.NewReader(bz))
assert.NilError(t, err)
assert.Equal(t, 0, cdc.Compare(x, y))
return bz
}
func TestUnsupportedFields(t *testing.T) {
_, err := ormfield.GetCodec(nil, false)
assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
_, err = ormfield.GetCodec(testutil.GetTestField("repeated"), false)
assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
_, err = ormfield.GetCodec(testutil.GetTestField("map"), false)
assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
_, err = ormfield.GetCodec(testutil.GetTestField("msg"), false)
assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
_, err = ormfield.GetCodec(testutil.GetTestField("oneof"), false)
assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
}
func TestNTBytesTooLong(t *testing.T) {
cdc, err := ormfield.GetCodec(testutil.GetTestField("bz"), true)
assert.NilError(t, err)
buf := &bytes.Buffer{}
bz := protoreflect.ValueOfBytes(make([]byte, 256))
assert.ErrorContains(t, cdc.Encode(bz, buf), ormerrors.BytesFieldTooLong.Error())
_, err = cdc.ComputeBufferSize(bz)
assert.ErrorContains(t, err, ormerrors.BytesFieldTooLong.Error())
}
+69
View File
@@ -0,0 +1,69 @@
package ormfield
import (
io "io"
"google.golang.org/protobuf/reflect/protoreflect"
)
var (
durationSecondsField = durationMsgType.Descriptor().Fields().ByName("seconds")
durationNanosField = durationMsgType.Descriptor().Fields().ByName("nanos")
)
func getDurationSecondsAndNanos(value protoreflect.Value) (protoreflect.Value, protoreflect.Value) {
msg := value.Message()
return msg.Get(durationSecondsField), msg.Get(durationNanosField)
}
// DurationCodec encodes a google.protobuf.Duration value as 12 bytes using
// Int64Codec for seconds followed by Int32Codec for nanos. This allows for
// sorted iteration.
type DurationCodec struct{}
func (d DurationCodec) Decode(r Reader) (protoreflect.Value, error) {
seconds, err := int64Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
nanos, err := int32Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
msg := durationMsgType.New()
msg.Set(durationSecondsField, seconds)
msg.Set(durationNanosField, nanos)
return protoreflect.ValueOfMessage(msg), nil
}
func (d DurationCodec) Encode(value protoreflect.Value, w io.Writer) error {
seconds, nanos := getDurationSecondsAndNanos(value)
err := int64Codec.Encode(seconds, w)
if err != nil {
return err
}
return int32Codec.Encode(nanos, w)
}
func (d DurationCodec) Compare(v1, v2 protoreflect.Value) int {
s1, n1 := getDurationSecondsAndNanos(v1)
s2, n2 := getDurationSecondsAndNanos(v2)
c := compareInt(s1, s2)
if c != 0 {
return c
} else {
return compareInt(n1, n2)
}
}
func (d DurationCodec) IsOrdered() bool {
return true
}
func (d DurationCodec) FixedBufferSize() int {
return 12
}
func (d DurationCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
return d.FixedBufferSize(), nil
}
+48
View File
@@ -0,0 +1,48 @@
package ormfield
import (
"encoding/binary"
io "io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// EnumCodec encodes enum values as varints.
type EnumCodec struct{}
func (e EnumCodec) Decode(r Reader) (protoreflect.Value, error) {
x, err := binary.ReadVarint(r)
return protoreflect.ValueOfEnum(protoreflect.EnumNumber(x)), err
}
func (e EnumCodec) Encode(value protoreflect.Value, w io.Writer) error {
x := value.Enum()
buf := make([]byte, binary.MaxVarintLen32)
n := binary.PutVarint(buf, int64(x))
_, err := w.Write(buf[:n])
return err
}
func (e EnumCodec) Compare(v1, v2 protoreflect.Value) int {
x := v1.Enum()
y := v2.Enum()
if x == y {
return 0
} else if x < y {
return -1
} else {
return 1
}
}
func (e EnumCodec) IsOrdered() bool {
return false
}
func (e EnumCodec) FixedBufferSize() int {
return binary.MaxVarintLen32
}
func (e EnumCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
return e.FixedBufferSize(), nil
}
+47
View File
@@ -0,0 +1,47 @@
package ormfield
import (
"encoding/binary"
io "io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// Int32Codec encodes 32-bit integers as big-endian unsigned 32-bit integers
// by adding the maximum value of int32 (2147583647) + 1 before encoding so
// that these values can be used for ordered iteration.
type Int32Codec struct{}
var int32Codec = Int32Codec{}
const int32Max = 2147483647
const int32Offset = int32Max + 1
func (i Int32Codec) Decode(r Reader) (protoreflect.Value, error) {
var x uint32
err := binary.Read(r, binary.BigEndian, &x)
y := int64(x) - int32Offset
return protoreflect.ValueOfInt32(int32(y)), err
}
func (i Int32Codec) Encode(value protoreflect.Value, w io.Writer) error {
x := value.Int()
x += int32Offset
return binary.Write(w, binary.BigEndian, uint32(x))
}
func (i Int32Codec) Compare(v1, v2 protoreflect.Value) int {
return compareInt(v1, v2)
}
func (i Int32Codec) IsOrdered() bool {
return true
}
func (i Int32Codec) FixedBufferSize() int {
return 4
}
func (i Int32Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
return i.FixedBufferSize(), nil
}
+69
View File
@@ -0,0 +1,69 @@
package ormfield
import (
"encoding/binary"
io "io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// Int64Codec encodes 64-bit integers as big-endian unsigned 64-bit integers
// by adding the maximum value of int32 (9223372036854775807) + 1 before encoding so
// that these values can be used for ordered iteration.
type Int64Codec struct{}
var int64Codec = Int64Codec{}
const int64Max = 9223372036854775807
func (i Int64Codec) Decode(r Reader) (protoreflect.Value, error) {
var x uint64
err := binary.Read(r, binary.BigEndian, &x)
if x >= int64Max {
x = x - int64Max - 1
return protoreflect.ValueOfInt64(int64(x)), err
} else {
y := int64(x) - int64Max - 1
return protoreflect.ValueOfInt64(y), err
}
}
func (i Int64Codec) Encode(value protoreflect.Value, w io.Writer) error {
x := value.Int()
if x >= -1 {
y := uint64(x) + int64Max + 1
return binary.Write(w, binary.BigEndian, y)
} else {
x += int64Max
x += 1
return binary.Write(w, binary.BigEndian, uint64(x))
}
}
func (i Int64Codec) Compare(v1, v2 protoreflect.Value) int {
return compareInt(v1, v2)
}
func (i Int64Codec) IsOrdered() bool {
return true
}
func (i Int64Codec) FixedBufferSize() int {
return 8
}
func (i Int64Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
return i.FixedBufferSize(), nil
}
func compareInt(v1, v2 protoreflect.Value) int {
x := v1.Int()
y := v2.Int()
if x == y {
return 0
} else if x < y {
return -1
} else {
return 1
}
}
+87
View File
@@ -0,0 +1,87 @@
package ormfield
import (
"fmt"
"io"
"strings"
"google.golang.org/protobuf/reflect/protoreflect"
)
// StringCodec encodes strings as raw bytes.
type StringCodec struct{}
func (s StringCodec) FixedBufferSize() int {
return -1
}
func (s StringCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
return len(value.Interface().(string)), nil
}
func (s StringCodec) IsOrdered() bool {
return true
}
func (s StringCodec) Compare(v1, v2 protoreflect.Value) int {
return strings.Compare(v1.Interface().(string), v2.Interface().(string))
}
func (s StringCodec) Decode(r Reader) (protoreflect.Value, error) {
bz, err := io.ReadAll(r)
return protoreflect.ValueOfString(string(bz)), err
}
func (s StringCodec) Encode(value protoreflect.Value, w io.Writer) error {
_, err := w.Write([]byte(value.Interface().(string)))
return err
}
// NonTerminalStringCodec encodes strings as null-terminated raw bytes. Null
// values within strings will produce an error.
type NonTerminalStringCodec struct{}
func (s NonTerminalStringCodec) FixedBufferSize() int {
return -1
}
func (s NonTerminalStringCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
return len(value.Interface().(string)) + 1, nil
}
func (s NonTerminalStringCodec) IsOrdered() bool {
return true
}
func (s NonTerminalStringCodec) Compare(v1, v2 protoreflect.Value) int {
return strings.Compare(v1.Interface().(string), v2.Interface().(string))
}
func (s NonTerminalStringCodec) Decode(r Reader) (protoreflect.Value, error) {
var bz []byte
for {
b, err := r.ReadByte()
if b == 0 || err == io.EOF {
return protoreflect.ValueOfString(string(bz)), err
}
bz = append(bz, b)
}
}
func (s NonTerminalStringCodec) Encode(value protoreflect.Value, w io.Writer) error {
str := value.Interface().(string)
bz := []byte(str)
for _, b := range bz {
if b == 0 {
return fmt.Errorf("illegal null terminator found in index string: %s", str)
}
}
_, err := w.Write([]byte(str))
if err != nil {
return err
}
_, err = w.Write(nullTerminator)
return err
}
var nullTerminator = []byte{0}
+69
View File
@@ -0,0 +1,69 @@
package ormfield
import (
"io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// TimestampCodec DurationCodec encodes a google.protobuf.Timestamp value as 12 bytes using
// Int64Codec for seconds followed by Int32Codec for nanos. This allows for
// sorted iteration.
type TimestampCodec struct{}
var (
timestampSecondsField = timestampMsgType.Descriptor().Fields().ByName("seconds")
timestampNanosField = timestampMsgType.Descriptor().Fields().ByName("nanos")
)
func getTimestampSecondsAndNanos(value protoreflect.Value) (protoreflect.Value, protoreflect.Value) {
msg := value.Message()
return msg.Get(timestampSecondsField), msg.Get(timestampNanosField)
}
func (t TimestampCodec) Decode(r Reader) (protoreflect.Value, error) {
seconds, err := int64Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
nanos, err := int32Codec.Decode(r)
if err != nil {
return protoreflect.Value{}, err
}
msg := timestampMsgType.New()
msg.Set(timestampSecondsField, seconds)
msg.Set(timestampNanosField, nanos)
return protoreflect.ValueOfMessage(msg), nil
}
func (t TimestampCodec) Encode(value protoreflect.Value, w io.Writer) error {
seconds, nanos := getTimestampSecondsAndNanos(value)
err := int64Codec.Encode(seconds, w)
if err != nil {
return err
}
return int32Codec.Encode(nanos, w)
}
func (t TimestampCodec) Compare(v1, v2 protoreflect.Value) int {
s1, n1 := getTimestampSecondsAndNanos(v1)
s2, n2 := getTimestampSecondsAndNanos(v2)
c := compareInt(s1, s2)
if c != 0 {
return c
} else {
return compareInt(n1, n2)
}
}
func (t TimestampCodec) IsOrdered() bool {
return true
}
func (t TimestampCodec) FixedBufferSize() int {
return 12
}
func (t TimestampCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
return t.FixedBufferSize(), nil
}
+37
View File
@@ -0,0 +1,37 @@
package ormfield
import (
"encoding/binary"
"io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// Uint32Codec encodes uint32 values as 4-byte big-endian integers.
type Uint32Codec struct{}
func (u Uint32Codec) FixedBufferSize() int {
return 4
}
func (u Uint32Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
return u.FixedBufferSize(), nil
}
func (u Uint32Codec) IsOrdered() bool {
return true
}
func (u Uint32Codec) Compare(v1, v2 protoreflect.Value) int {
return compareUint(v1, v2)
}
func (u Uint32Codec) Decode(r Reader) (protoreflect.Value, error) {
var x uint32
err := binary.Read(r, binary.BigEndian, &x)
return protoreflect.ValueOfUint32(x), err
}
func (u Uint32Codec) Encode(value protoreflect.Value, w io.Writer) error {
return binary.Write(w, binary.BigEndian, uint32(value.Uint()))
}
+49
View File
@@ -0,0 +1,49 @@
package ormfield
import (
"encoding/binary"
"io"
"google.golang.org/protobuf/reflect/protoreflect"
)
// Uint64Codec encodes uint64 values as 8-byte big-endian integers.
type Uint64Codec struct{}
func (u Uint64Codec) FixedBufferSize() int {
return 8
}
func (u Uint64Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
return u.FixedBufferSize(), nil
}
func (u Uint64Codec) IsOrdered() bool {
return true
}
func (u Uint64Codec) Compare(v1, v2 protoreflect.Value) int {
return compareUint(v1, v2)
}
func (u Uint64Codec) Decode(r Reader) (protoreflect.Value, error) {
var x uint64
err := binary.Read(r, binary.BigEndian, &x)
return protoreflect.ValueOfUint64(x), err
}
func (u Uint64Codec) Encode(value protoreflect.Value, w io.Writer) error {
return binary.Write(w, binary.BigEndian, value.Uint())
}
func compareUint(v1, v2 protoreflect.Value) int {
x := v1.Uint()
y := v2.Uint()
if x == y {
return 0
} else if x < y {
return -1
} else {
return 1
}
}