feat(orm): add ormkv.KeyCodec (#10640)

* feat(orm): add KeyCodec

* WIP

* code coverage

* add DefaultValue test

* fix range key check

* revert DefaultValue

* fix range check

* Update orm/encoding/ormkv/key_codec.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/key_codec.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/key_codec.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/util.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/key_codec.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/key_codec.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/key_codec.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* address review comments

* address review comments

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>
This commit is contained in:
Aaron Craelius
2021-11-30 16:27:17 -05:00
committed by GitHub
co-authored by Tyler
parent c4bedf8a56
commit cf6ace5a1c
6 changed files with 693 additions and 4 deletions
+271
View File
@@ -0,0 +1,271 @@
package ormkv
import (
"bytes"
"io"
"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
"google.golang.org/protobuf/reflect/protoreflect"
"github.com/cosmos/cosmos-sdk/orm/encoding/ormfield"
)
type KeyCodec struct {
fixedSize int
variableSizers []struct {
cdc ormfield.Codec
i int
}
prefix []byte
fieldDescriptors []protoreflect.FieldDescriptor
fieldCodecs []ormfield.Codec
}
// NewKeyCodec returns a new KeyCodec with the provided prefix and
// codecs for the provided fields.
func NewKeyCodec(prefix []byte, fieldDescriptors []protoreflect.FieldDescriptor) (*KeyCodec, error) {
n := len(fieldDescriptors)
var fieldCodecs []ormfield.Codec
var variableSizers []struct {
cdc ormfield.Codec
i int
}
fixedSize := 0
names := make([]protoreflect.Name, len(fieldDescriptors))
for i := 0; i < n; i++ {
nonTerminal := i != n-1
field := fieldDescriptors[i]
cdc, err := ormfield.GetCodec(field, nonTerminal)
if err != nil {
return nil, err
}
if x := cdc.FixedBufferSize(); x > 0 {
fixedSize += x
} else {
variableSizers = append(variableSizers, struct {
cdc ormfield.Codec
i int
}{cdc, i})
}
fieldCodecs = append(fieldCodecs, cdc)
names[i] = field.Name()
}
return &KeyCodec{
fieldCodecs: fieldCodecs,
fieldDescriptors: fieldDescriptors,
prefix: prefix,
fixedSize: fixedSize,
variableSizers: variableSizers,
}, nil
}
// Encode encodes the values assuming that they correspond to the fields
// specified for the key. If the array of values is shorter than the
// number of fields in the key, a partial "prefix" key will be encoded
// which can be used for constructing a prefix iterator.
func (cdc *KeyCodec) Encode(values []protoreflect.Value) ([]byte, error) {
sz, err := cdc.ComputeBufferSize(values)
if err != nil {
return nil, err
}
w := bytes.NewBuffer(make([]byte, 0, sz))
if _, err = w.Write(cdc.prefix); err != nil {
return nil, err
}
n := len(values)
if n > len(cdc.fieldCodecs) {
return nil, ormerrors.IndexOutOfBounds.Wrapf("cannot encode %d values into %d fields", n, len(cdc.fieldCodecs))
}
for i := 0; i < n; i++ {
if err = cdc.fieldCodecs[i].Encode(values[i], w); err != nil {
return nil, err
}
}
return w.Bytes(), nil
}
// GetValues extracts the values specified by the key fields from the message.
func (cdc *KeyCodec) GetValues(message protoreflect.Message) []protoreflect.Value {
res := make([]protoreflect.Value, len(cdc.fieldDescriptors))
for i, f := range cdc.fieldDescriptors {
res[i] = message.Get(f)
}
return res
}
// Decode decodes the values in the key specified by the reader. If the
// provided key is a prefix key, the values that could be decoded will
// be returned with io.EOF as the error.
func (cdc *KeyCodec) Decode(r *bytes.Reader) ([]protoreflect.Value, error) {
if err := skipPrefix(r, cdc.prefix); err != nil {
return nil, err
}
n := len(cdc.fieldCodecs)
values := make([]protoreflect.Value, 0, n)
for i := 0; i < n; i++ {
value, err := cdc.fieldCodecs[i].Decode(r)
if err == io.EOF {
return values, err
} else if err != nil {
return nil, err
}
values = append(values, value)
}
return values, nil
}
// EncodeFromMessage combines GetValues and Encode.
func (cdc *KeyCodec) EncodeFromMessage(message protoreflect.Message) ([]protoreflect.Value, []byte, error) {
values := cdc.GetValues(message)
bz, err := cdc.Encode(values)
return values, bz, err
}
// IsFullyOrdered returns true if all fields are also ordered.
func (cdc *KeyCodec) IsFullyOrdered() bool {
for _, p := range cdc.fieldCodecs {
if !p.IsOrdered() {
return false
}
}
return true
}
// CompareValues compares the provided values which must correspond to the
// fields in this key. Prefix keys of different lengths are supported but the
// function will panic if either array is too long. A negative value is returned
// if values1 is less than values2, 0 is returned if the two arrays are equal,
// and a positive value is returned if values2 is greater.
func (cdc *KeyCodec) CompareValues(values1, values2 []protoreflect.Value) int {
j := len(values1)
k := len(values2)
n := j
if k < j {
n = k
}
if n > len(cdc.fieldCodecs) {
panic("array is too long")
}
var cmp int
for i := 0; i < n; i++ {
cmp = cdc.fieldCodecs[i].Compare(values1[i], values2[i])
// any non-equal parts determine our ordering
if cmp != 0 {
return cmp
}
}
// values are equal but arrays of different length
if j == k {
return 0
} else if j < k {
return -1
} else {
return 1
}
}
// ComputeBufferSize computes the required buffer size for the provided values
// which can represent a full or prefix key.
func (cdc KeyCodec) ComputeBufferSize(values []protoreflect.Value) (int, error) {
size := cdc.fixedSize
n := len(values)
for _, sz := range cdc.variableSizers {
// handle prefix key encoding case where don't need all the sizers
if sz.i >= n {
return size, nil
}
x, err := sz.cdc.ComputeBufferSize(values[sz.i])
if err != nil {
return 0, err
}
size += x
}
return size, nil
}
// SetValues sets the provided values on the message which must correspond
// exactly to the field descriptors for this key. Prefix keys aren't
// supported.
func (cdc *KeyCodec) SetValues(message protoreflect.Message, values []protoreflect.Value) {
for i, f := range cdc.fieldDescriptors {
message.Set(f, values[i])
}
}
// CheckValidRangeIterationKeys checks if the start and end key prefixes are valid
// for range iteration meaning that for each non-equal field in the prefixes
// those field types support ordered iteration. If start or end is longer than
// the other, the omitted values will function as the minimum and maximum
// values of that type respectively.
func (cdc KeyCodec) CheckValidRangeIterationKeys(start, end []protoreflect.Value) error {
lenStart := len(start)
shortest := lenStart
longest := lenStart
lenEnd := len(end)
if lenEnd < shortest {
shortest = lenEnd
} else {
longest = lenEnd
}
if longest > len(cdc.fieldCodecs) {
return ormerrors.IndexOutOfBounds
}
i := 0
var cmp int
for ; i < shortest; i++ {
fieldCdc := cdc.fieldCodecs[i]
x := start[i]
y := end[i]
cmp = fieldCdc.Compare(x, y)
if cmp > 0 {
return ormerrors.InvalidRangeIterationKeys.Wrapf(
"start must be before end for field %s",
cdc.fieldDescriptors[i].FullName(),
)
} else if !fieldCdc.IsOrdered() && cmp != 0 {
descriptor := cdc.fieldDescriptors[i]
return ormerrors.InvalidRangeIterationKeys.Wrapf(
"field %s of kind %s doesn't support ordered range iteration",
descriptor.FullName(),
descriptor.Kind(),
)
} else if cmp < 0 {
break
}
}
// the last prefix value must not be equal if the key lengths are the same
if lenStart == lenEnd {
if cmp == 0 {
return ormerrors.InvalidRangeIterationKeys
}
} else {
// check any remaining values in start or end
for j := i; j < longest; j++ {
if !cdc.fieldCodecs[j].IsOrdered() {
return ormerrors.InvalidRangeIterationKeys.Wrapf(
"field %s of kind %s doesn't support ordered range iteration",
cdc.fieldDescriptors[j].FullName(),
cdc.fieldDescriptors[j].Kind(),
)
}
}
}
return nil
}
+337
View File
@@ -0,0 +1,337 @@
package ormkv_test
import (
"bytes"
"io"
"testing"
"github.com/cosmos/cosmos-sdk/orm/internal/testpb"
"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
"google.golang.org/protobuf/reflect/protoreflect"
"gotest.tools/v3/assert"
"pgregory.net/rapid"
"github.com/cosmos/cosmos-sdk/orm/internal/testutil"
)
func TestKeyCodec(t *testing.T) {
rapid.Check(t, func(t *rapid.T) {
key := testutil.TestKeyCodecGen.Draw(t, "key").(testutil.TestKeyCodec)
for i := 0; i < 100; i++ {
keyValues := key.Draw(t, "values")
bz1 := assertEncDecKey(t, key, keyValues)
if key.Codec.IsFullyOrdered() {
// check if ordered keys have ordered encodings
keyValues2 := key.Draw(t, "values2")
bz2 := assertEncDecKey(t, key, keyValues2)
// bytes comparison should equal comparison of values
assert.Equal(t, key.Codec.CompareValues(keyValues, keyValues2), bytes.Compare(bz1, bz2))
}
}
})
}
func assertEncDecKey(t *rapid.T, key testutil.TestKeyCodec, keyValues []protoreflect.Value) []byte {
bz, err := key.Codec.Encode(keyValues)
assert.NilError(t, err)
keyValues2, err := key.Codec.Decode(bytes.NewReader(bz))
assert.NilError(t, err)
assert.Equal(t, 0, key.Codec.CompareValues(keyValues, keyValues2))
return bz
}
func TestCompareValues(t *testing.T) {
cdc, err := ormkv.NewKeyCodec(nil, []protoreflect.FieldDescriptor{
testutil.GetTestField("u32"),
testutil.GetTestField("str"),
testutil.GetTestField("i32"),
})
assert.NilError(t, err)
tests := []struct {
name string
values1 []protoreflect.Value
values2 []protoreflect.Value
expect int
validRange bool
}{
{
"eq",
ValuesOf(uint32(0), "abc", int32(-3)),
ValuesOf(uint32(0), "abc", int32(-3)),
0,
false,
},
{
"eq prefix 0",
ValuesOf(),
ValuesOf(),
0,
false,
},
{
"eq prefix 1",
ValuesOf(uint32(0)),
ValuesOf(uint32(0)),
0,
false,
},
{
"eq prefix 2",
ValuesOf(uint32(0), "abc"),
ValuesOf(uint32(0), "abc"),
0,
false,
},
{
"lt1",
ValuesOf(uint32(0), "abc", int32(-3)),
ValuesOf(uint32(1), "abc", int32(-3)),
-1,
true,
},
{
"lt2",
ValuesOf(uint32(1), "abb", int32(-3)),
ValuesOf(uint32(1), "abc", int32(-3)),
-1,
true,
},
{
"lt3",
ValuesOf(uint32(1), "abb", int32(-4)),
ValuesOf(uint32(1), "abb", int32(-3)),
-1,
true,
},
{
"less prefix 0",
ValuesOf(),
ValuesOf(uint32(1), "abb", int32(-4)),
-1,
true,
},
{
"less prefix 1",
ValuesOf(uint32(1)),
ValuesOf(uint32(1), "abb", int32(-4)),
-1,
true,
},
{
"less prefix 2",
ValuesOf(uint32(1), "abb"),
ValuesOf(uint32(1), "abb", int32(-4)),
-1,
true,
},
{
"gt1",
ValuesOf(uint32(2), "abb", int32(-4)),
ValuesOf(uint32(1), "abb", int32(-4)),
1,
false,
},
{
"gt2",
ValuesOf(uint32(2), "abc", int32(-4)),
ValuesOf(uint32(2), "abb", int32(-4)),
1,
false,
},
{
"gt3",
ValuesOf(uint32(2), "abc", int32(1)),
ValuesOf(uint32(2), "abc", int32(-3)),
1,
false,
},
{
"gt prefix 0",
ValuesOf(uint32(2), "abc", int32(-3)),
ValuesOf(),
1,
true,
},
{
"gt prefix 1",
ValuesOf(uint32(2), "abc", int32(-3)),
ValuesOf(uint32(2)),
1,
true,
},
{
"gt prefix 2",
ValuesOf(uint32(2), "abc", int32(-3)),
ValuesOf(uint32(2), "abc"),
1,
true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(
t, test.expect,
cdc.CompareValues(test.values1, test.values2),
)
// CheckValidRangeIterationKeys should give comparable results
err := cdc.CheckValidRangeIterationKeys(test.values1, test.values2)
if test.validRange {
assert.NilError(t, err)
} else {
assert.ErrorContains(t, err, "")
}
})
}
}
func ValuesOf(values ...interface{}) []protoreflect.Value {
n := len(values)
res := make([]protoreflect.Value, n)
for i := 0; i < n; i++ {
res[i] = protoreflect.ValueOf(values[i])
}
return res
}
func TestDecodePrefixKey(t *testing.T) {
cdc, err := ormkv.NewKeyCodec(nil, []protoreflect.FieldDescriptor{
testutil.GetTestField("u32"),
testutil.GetTestField("str"),
testutil.GetTestField("bz"),
testutil.GetTestField("i32"),
})
assert.NilError(t, err)
tests := []struct {
name string
values []protoreflect.Value
}{
{
"1",
ValuesOf(uint32(5), "abc"),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
bz, err := cdc.Encode(test.values)
assert.NilError(t, err)
values, err := cdc.Decode(bytes.NewReader(bz))
assert.ErrorType(t, err, io.EOF)
assert.Equal(t, 0, cdc.CompareValues(test.values, values))
})
}
}
func TestValidRangeIterationKeys(t *testing.T) {
cdc, err := ormkv.NewKeyCodec(nil, []protoreflect.FieldDescriptor{
testutil.GetTestField("u32"),
testutil.GetTestField("str"),
testutil.GetTestField("bz"),
testutil.GetTestField("i32"),
})
assert.NilError(t, err)
tests := []struct {
name string
values1 []protoreflect.Value
values2 []protoreflect.Value
expectErr bool
}{
{
"1 eq",
ValuesOf(uint32(0)),
ValuesOf(uint32(0)),
true,
},
{
"1 lt",
ValuesOf(uint32(0)),
ValuesOf(uint32(1)),
false,
},
{
"1 gt",
ValuesOf(uint32(1)),
ValuesOf(uint32(0)),
true,
},
{
"1,2 lt",
ValuesOf(uint32(0)),
ValuesOf(uint32(0), "abc"),
false,
},
{
"1,2 gt",
ValuesOf(uint32(0), "abc"),
ValuesOf(uint32(0)),
false,
},
{
"1,2,3",
ValuesOf(uint32(0)),
ValuesOf(uint32(0), "abc", []byte{1, 2}),
true,
},
{
"1,2,3,4 lt",
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(-1)),
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(1)),
false,
},
{
"too long",
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(-1)),
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(1), int32(1)),
true,
},
{
"1,2,3,4 eq",
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(1)),
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(1)),
true,
},
{
"1,2,3,4 bz err",
ValuesOf(uint32(0), "abc", []byte{1, 2}, int32(-1)),
ValuesOf(uint32(0), "abc", []byte{1, 2, 3}, int32(1)),
true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := cdc.CheckValidRangeIterationKeys(test.values1, test.values2)
if test.expectErr {
assert.ErrorContains(t, err, "")
} else {
assert.NilError(t, err)
}
})
}
}
func TestGetSet(t *testing.T) {
cdc, err := ormkv.NewKeyCodec(nil, []protoreflect.FieldDescriptor{
testutil.GetTestField("u32"),
testutil.GetTestField("str"),
testutil.GetTestField("i32"),
})
assert.NilError(t, err)
var a testpb.A
values := ValuesOf(uint32(4), "abc", int32(1))
cdc.SetValues(a.ProtoReflect(), values)
values2 := cdc.GetValues(a.ProtoReflect())
assert.Equal(t, 0, cdc.CompareValues(values, values2))
bz, err := cdc.Encode(values)
assert.NilError(t, err)
values3, bz2, err := cdc.EncodeFromMessage(a.ProtoReflect())
assert.NilError(t, err)
assert.Equal(t, 0, cdc.CompareValues(values, values3))
assert.Assert(t, bytes.Equal(bz, bz2))
}
+17
View File
@@ -0,0 +1,17 @@
package ormkv
import (
"bytes"
"io"
)
func skipPrefix(r *bytes.Reader, prefix []byte) error {
n := len(prefix)
if n > 0 {
// we skip checking the prefix for performance reasons because we assume
// that it was checked by the caller
_, err := r.Seek(int64(n), io.SeekCurrent);
return err
}
return nil
}