feat(orm): add module db (#10991)

## Description This PR adds a `ModuleDB` interface which can be used directly by Cosmos SDK modules. A simplified bank example with Mint/Send/Burn functionality against Balance and Supply tables is included in the tests. This PR also: * adds simplified `Get` and `Has` methods to `Table` which use the primary key values in the message instead of `...interface{}` * adds a stable deterministic proto JSON marshaler and updates the `Entry.String` methods to use it because the golden tests are not deterministic without this. This code is currently internal but can be extracted to a public `codec` or `cosmos-proto` package eventually. --- ### Author Checklist *All items are required. Please add a note to the item if the item is not applicable and please add links to any relevant follow up issues.* I have... - [ ] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] added `!` to the type prefix if API or client breaking change - [ ] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting)) - [ ] provided a link to the relevant issue or specification - [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules) - [ ] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing) - [ ] added a changelog entry to `CHANGELOG.md` - [ ] included comments for [documenting Go code](https://blog.golang.org/godoc) - [ ] updated the relevant documentation or specification - [ ] reviewed "Files changed" and left comments if necessary - [ ] confirmed all CI checks have passed ### Reviewers Checklist *All items are required. Please add a note if the item is not applicable and please add your handle next to the items reviewed if you only reviewed selected items.* I have... - [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] confirmed `!` in the type prefix if API or client breaking change - [ ] confirmed all author checklist items have been addressed - [ ] reviewed state machine logic - [ ] reviewed API design and naming - [ ] reviewed documentation is accurate - [ ] reviewed tests and test coverage - [ ] manually tested (if applicable)
2022-01-22 03:13:43 +00:00
parent 87bb06c9fc
commit 6ea2049944
21 changed files with 2346 additions and 323 deletions
@@ -0,0 +1,125 @@
+package ormdb
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"math"
+
+	"google.golang.org/protobuf/reflect/protoregistry"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/encodeutil"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	"github.com/cosmos/cosmos-sdk/orm/model/ormtable"
+)
+
+type fileDescriptorDBOptions struct {
+	Prefix         []byte
+	ID             uint32
+	TypeResolver   ormtable.TypeResolver
+	JSONValidator  func(proto.Message) error
+	GetBackend     func(context.Context) (ormtable.Backend, error)
+	GetReadBackend func(context.Context) (ormtable.ReadBackend, error)
+}
+
+type fileDescriptorDB struct {
+	id             uint32
+	prefix         []byte
+	tablesById     map[uint32]ormtable.Table
+	tablesByName   map[protoreflect.FullName]ormtable.Table
+	fileDescriptor protoreflect.FileDescriptor
+}
+
+func newFileDescriptorDB(fileDescriptor protoreflect.FileDescriptor, options fileDescriptorDBOptions) (*fileDescriptorDB, error) {
+	prefix := encodeutil.AppendVarUInt32(options.Prefix, options.ID)
+
+	schema := &fileDescriptorDB{
+		id:             options.ID,
+		prefix:         prefix,
+		tablesById:     map[uint32]ormtable.Table{},
+		tablesByName:   map[protoreflect.FullName]ormtable.Table{},
+		fileDescriptor: fileDescriptor,
+	}
+
+	resolver := options.TypeResolver
+	if resolver == nil {
+		resolver = protoregistry.GlobalTypes
+	}
+
+	messages := fileDescriptor.Messages()
+	n := messages.Len()
+	for i := 0; i < n; i++ {
+		messageDescriptor := messages.Get(i)
+		tableName := messageDescriptor.FullName()
+		messageType, err := resolver.FindMessageByName(tableName)
+		if err != nil {
+			return nil, err
+		}
+
+		table, err := ormtable.Build(ormtable.Options{
+			Prefix:         prefix,
+			MessageType:    messageType,
+			TypeResolver:   resolver,
+			JSONValidator:  options.JSONValidator,
+			GetReadBackend: options.GetReadBackend,
+			GetBackend:     options.GetBackend,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		id := table.ID()
+		if _, ok := schema.tablesById[id]; ok {
+			return nil, ormerrors.InvalidTableId.Wrapf("duplicate ID %d for %s", id, tableName)
+		}
+		schema.tablesById[id] = table
+
+		if _, ok := schema.tablesByName[tableName]; ok {
+			return nil, ormerrors.InvalidTableDefinition.Wrapf("duplicate table %s", tableName)
+		}
+		schema.tablesByName[tableName] = table
+	}
+
+	return schema, nil
+}
+
+func (f fileDescriptorDB) DecodeEntry(k, v []byte) (ormkv.Entry, error) {
+	r := bytes.NewReader(k)
+	err := encodeutil.SkipPrefix(r, f.prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	id, err := binary.ReadUvarint(r)
+	if err != nil {
+		return nil, err
+	}
+
+	if id > math.MaxUint32 {
+		return nil, ormerrors.UnexpectedDecodePrefix.Wrapf("uint32 varint id out of range %d", id)
+	}
+
+	table, ok := f.tablesById[uint32(id)]
+	if !ok {
+		return nil, ormerrors.UnexpectedDecodePrefix.Wrapf("can't find table with id %d", id)
+	}
+
+	return table.DecodeEntry(k, v)
+}
+
+func (f fileDescriptorDB) EncodeEntry(entry ormkv.Entry) (k, v []byte, err error) {
+	table, ok := f.tablesByName[entry.GetTableName()]
+	if !ok {
+		return nil, nil, ormerrors.BadDecodeEntry.Wrapf("can't find table %s", entry.GetTableName())
+	}
+
+	return table.EncodeEntry(entry)
+}
+
+var _ ormkv.EntryCodec = fileDescriptorDB{}
@@ -0,0 +1,162 @@
+package ormdb
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"math"
+
+	"google.golang.org/protobuf/reflect/protodesc"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/encodeutil"
+
+	"google.golang.org/protobuf/proto"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
+	"github.com/cosmos/cosmos-sdk/orm/model/ormtable"
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+)
+
+// ModuleSchema describes the ORM schema for a module.
+type ModuleSchema struct {
+	// FileDescriptors are the file descriptors that contain ORM tables to use in this schema.
+	// Each file descriptor must have an unique non-zero uint32 ID associated with it.
+	FileDescriptors map[uint32]protoreflect.FileDescriptor
+
+	// Prefix is an optional prefix to prepend to all keys. It is recommended
+	// to leave it empty.
+	Prefix []byte
+}
+
+// ModuleDB defines the ORM database type to be used by modules.
+type ModuleDB interface {
+	ormkv.EntryCodec
+
+	// GetTable returns the table for the provided message type or nil.
+	GetTable(message proto.Message) ormtable.Table
+}
+
+type moduleDB struct {
+	prefix       []byte
+	filesById    map[uint32]*fileDescriptorDB
+	tablesByName map[protoreflect.FullName]ormtable.Table
+}
+
+// ModuleDBOptions are options for constructing a ModuleDB.
+type ModuleDBOptions struct {
+
+	// TypeResolver is an optional type resolver to be used when unmarshaling
+	// protobuf messages. If it is nil, protoregistry.GlobalTypes will be used.
+	TypeResolver ormtable.TypeResolver
+
+	// FileResolver is an optional file resolver that can be used to retrieve
+	// pinned file descriptors that may be different from those available at
+	// runtime. The file descriptor versions returned by this resolver will be
+	// used instead of the ones provided at runtime by the ModuleSchema.
+	FileResolver protodesc.Resolver
+
+	// JSONValidator is an optional validator that can be used for validating
+	// messaging when using ValidateJSON. If it is nil, DefaultJSONValidator
+	// will be used
+	JSONValidator func(proto.Message) error
+
+	// GetBackend is the function used to retrieve the table backend.
+	// See ormtable.Options.GetBackend for more details.
+	GetBackend func(context.Context) (ormtable.Backend, error)
+
+	// GetReadBackend is the function used to retrieve a table read backend.
+	// See ormtable.Options.GetReadBackend for more details.
+	GetReadBackend func(context.Context) (ormtable.ReadBackend, error)
+}
+
+// NewModuleDB constructs a ModuleDB instance from the provided schema and options.
+func NewModuleDB(schema ModuleSchema, options ModuleDBOptions) (ModuleDB, error) {
+	prefix := schema.Prefix
+	db := &moduleDB{
+		prefix:       prefix,
+		filesById:    map[uint32]*fileDescriptorDB{},
+		tablesByName: map[protoreflect.FullName]ormtable.Table{},
+	}
+
+	for id, fileDescriptor := range schema.FileDescriptors {
+		if id == 0 {
+			return nil, ormerrors.InvalidFileDescriptorID.Wrapf("for %s", fileDescriptor.Path())
+		}
+
+		opts := fileDescriptorDBOptions{
+			ID:             id,
+			Prefix:         prefix,
+			TypeResolver:   options.TypeResolver,
+			JSONValidator:  options.JSONValidator,
+			GetBackend:     options.GetBackend,
+			GetReadBackend: options.GetReadBackend,
+		}
+
+		if options.FileResolver != nil {
+			// if a FileResolver is provided, we use that to resolve the file
+			// and not the one provided as a different pinned file descriptor
+			// may have been provided
+			var err error
+			fileDescriptor, err = options.FileResolver.FindFileByPath(fileDescriptor.Path())
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		fdSchema, err := newFileDescriptorDB(fileDescriptor, opts)
+		if err != nil {
+			return nil, err
+		}
+
+		db.filesById[id] = fdSchema
+		for name, table := range fdSchema.tablesByName {
+			if _, ok := db.tablesByName[name]; ok {
+				return nil, ormerrors.UnexpectedError.Wrapf("duplicate table %s", name)
+			}
+
+			db.tablesByName[name] = table
+		}
+	}
+
+	return db, nil
+}
+
+func (m moduleDB) DecodeEntry(k, v []byte) (ormkv.Entry, error) {
+	r := bytes.NewReader(k)
+	err := encodeutil.SkipPrefix(r, m.prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	id, err := binary.ReadUvarint(r)
+	if err != nil {
+		return nil, err
+	}
+
+	if id > math.MaxUint32 {
+		return nil, ormerrors.UnexpectedDecodePrefix.Wrapf("uint32 varint id out of range %d", id)
+	}
+
+	fileSchema, ok := m.filesById[uint32(id)]
+	if !ok {
+		return nil, ormerrors.UnexpectedDecodePrefix.Wrapf("can't find FileDescriptor schema with id %d", id)
+	}
+
+	return fileSchema.DecodeEntry(k, v)
+}
+
+func (m moduleDB) EncodeEntry(entry ormkv.Entry) (k, v []byte, err error) {
+	tableName := entry.GetTableName()
+	table, ok := m.tablesByName[tableName]
+	if !ok {
+		return nil, nil, ormerrors.BadDecodeEntry.Wrapf("can't find table %s", tableName)
+	}
+
+	return table.EncodeEntry(entry)
+}
+
+func (m moduleDB) GetTable(message proto.Message) ormtable.Table {
+	return m.tablesByName[message.ProtoReflect().Descriptor().FullName()]
+}
@@ -0,0 +1,220 @@
+package ormdb_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/golden"
+
+	"github.com/cosmos/cosmos-sdk/orm/internal/testkv"
+	"github.com/cosmos/cosmos-sdk/orm/internal/testpb"
+	"github.com/cosmos/cosmos-sdk/orm/model/ormdb"
+	"github.com/cosmos/cosmos-sdk/orm/model/ormtable"
+)
+
+// These tests use a simulated bank keeper. Addresses and balances use
+// string and uint64 types respectively for simplicity.
+
+var TestBankSchema = ormdb.ModuleSchema{
+	FileDescriptors: map[uint32]protoreflect.FileDescriptor{
+		1: testpb.File_testpb_bank_proto,
+	},
+}
+
+type keeper struct {
+	balanceTable             ormtable.Table
+	balanceAddressDenomIndex ormtable.UniqueIndex
+	balanceDenomIndex        ormtable.Index
+
+	supplyTable      ormtable.Table
+	supplyDenomIndex ormtable.UniqueIndex
+}
+
+func (k keeper) Send(ctx context.Context, from, to, denom string, amount uint64) error {
+	err := k.safeSubBalance(ctx, from, denom, amount)
+	if err != nil {
+		return err
+	}
+
+	return k.addBalance(ctx, to, denom, amount)
+}
+
+func (k keeper) Mint(ctx context.Context, acct, denom string, amount uint64) error {
+	supply := &testpb.Supply{Denom: denom}
+	_, err := k.supplyTable.Get(ctx, supply)
+	if err != nil {
+		return err
+	}
+
+	supply.Amount = supply.Amount + amount
+	err = k.supplyTable.Save(ctx, supply)
+	if err != nil {
+		return err
+	}
+
+	return k.addBalance(ctx, acct, denom, amount)
+}
+
+func (k keeper) Burn(ctx context.Context, acct, denom string, amount uint64) error {
+	supply := &testpb.Supply{Denom: denom}
+	found, err := k.supplyTable.Get(ctx, supply)
+	if err != nil {
+		return err
+	}
+
+	if !found {
+		return fmt.Errorf("no supply for %s", denom)
+	}
+
+	if amount > supply.Amount {
+		return fmt.Errorf("insufficient supply")
+	}
+
+	supply.Amount = supply.Amount - amount
+
+	if supply.Amount == 0 {
+		err = k.supplyTable.Delete(ctx, supply)
+	} else {
+		err = k.supplyTable.Save(ctx, supply)
+	}
+	if err != nil {
+		return err
+	}
+
+	return k.safeSubBalance(ctx, acct, denom, amount)
+}
+
+func (k keeper) Balance(ctx context.Context, acct, denom string) (uint64, error) {
+	balance := &testpb.Balance{Address: acct, Denom: denom}
+	_, err := k.balanceTable.Get(ctx, balance)
+	return balance.Amount, err
+}
+
+func (k keeper) Supply(ctx context.Context, denom string) (uint64, error) {
+	supply := &testpb.Supply{Denom: denom}
+	_, err := k.supplyTable.Get(ctx, supply)
+	return supply.Amount, err
+}
+
+func (k keeper) addBalance(ctx context.Context, acct, denom string, amount uint64) error {
+	balance := &testpb.Balance{Address: acct, Denom: denom}
+	_, err := k.balanceTable.Get(ctx, balance)
+	if err != nil {
+		return err
+	}
+
+	balance.Amount = balance.Amount + amount
+	return k.balanceTable.Save(ctx, balance)
+}
+
+func (k keeper) safeSubBalance(ctx context.Context, acct, denom string, amount uint64) error {
+	balance := &testpb.Balance{Address: acct, Denom: denom}
+	found, err := k.balanceTable.Get(ctx, balance)
+	if err != nil {
+		return err
+	}
+
+	if !found {
+		return fmt.Errorf("acct %x has no balance for %s", acct, denom)
+	}
+
+	if amount > balance.Amount {
+		return fmt.Errorf("insufficient funds")
+	}
+
+	balance.Amount = balance.Amount - amount
+
+	if balance.Amount == 0 {
+		return k.balanceTable.Delete(ctx, balance)
+	} else {
+		return k.balanceTable.Save(ctx, balance)
+	}
+}
+
+func newKeeper(db ormdb.ModuleDB) keeper {
+	k := keeper{
+		balanceTable: db.GetTable(&testpb.Balance{}),
+		supplyTable:  db.GetTable(&testpb.Supply{}),
+	}
+
+	k.balanceAddressDenomIndex = k.balanceTable.GetUniqueIndex("address,denom")
+	k.balanceDenomIndex = k.balanceTable.GetIndex("denom")
+	k.supplyDenomIndex = k.supplyTable.GetUniqueIndex("denom")
+
+	return k
+}
+
+func TestModuleDB(t *testing.T) {
+	// create db & debug context
+	db, err := ormdb.NewModuleDB(TestBankSchema, ormdb.ModuleDBOptions{})
+	assert.NilError(t, err)
+	debugBuf := &strings.Builder{}
+	store := testkv.NewDebugBackend(
+		testkv.NewSharedMemBackend(),
+		&testkv.EntryCodecDebugger{
+			EntryCodec: db,
+			Print:      func(s string) { debugBuf.WriteString(s + "\n") },
+		},
+	)
+	ctx := ormtable.WrapContextDefault(store)
+
+	// create keeper
+	k := newKeeper(db)
+	assert.Assert(t, k.balanceTable != nil)
+	assert.Assert(t, k.balanceAddressDenomIndex != nil)
+	assert.Assert(t, k.balanceDenomIndex != nil)
+	assert.Assert(t, k.supplyTable != nil)
+	assert.Assert(t, k.supplyDenomIndex != nil)
+
+	// mint coins
+	denom := "foo"
+	acct1 := "bob"
+	err = k.Mint(ctx, acct1, denom, 100)
+	assert.NilError(t, err)
+	bal, err := k.Balance(ctx, acct1, denom)
+	assert.NilError(t, err)
+	assert.Equal(t, uint64(100), bal)
+	supply, err := k.Supply(ctx, denom)
+	assert.NilError(t, err)
+	assert.Equal(t, uint64(100), supply)
+
+	// send coins
+	acct2 := "sally"
+	err = k.Send(ctx, acct1, acct2, denom, 30)
+	bal, err = k.Balance(ctx, acct1, denom)
+	assert.NilError(t, err)
+	assert.Equal(t, uint64(70), bal)
+	bal, err = k.Balance(ctx, acct2, denom)
+	assert.NilError(t, err)
+	assert.Equal(t, uint64(30), bal)
+
+	// burn coins
+	err = k.Burn(ctx, acct2, denom, 3)
+	bal, err = k.Balance(ctx, acct2, denom)
+	assert.NilError(t, err)
+	assert.Equal(t, uint64(27), bal)
+	supply, err = k.Supply(ctx, denom)
+	assert.NilError(t, err)
+	assert.Equal(t, uint64(97), supply)
+
+	// check debug output
+	golden.Assert(t, debugBuf.String(), "bank_scenario.golden")
+
+	// check decode & encode
+	it, err := store.CommitmentStore().Iterator(nil, nil)
+	assert.NilError(t, err)
+	for it.Valid() {
+		entry, err := db.DecodeEntry(it.Key(), it.Value())
+		assert.NilError(t, err)
+		k, v, err := db.EncodeEntry(entry)
+		assert.NilError(t, err)
+		assert.Assert(t, bytes.Equal(k, it.Key()))
+		assert.Assert(t, bytes.Equal(v, it.Value()))
+		it.Next()
+	}
+}
@@ -0,0 +1,58 @@
+GET 010200666f6f 
+    PK testpb.Supply foo -> {"denom":"foo"}
+GET 010200666f6f 
+    PK testpb.Supply foo -> {"denom":"foo"}
+ORM INSERT testpb.Supply {"denom":"foo","amount":100}
+SET 010200666f6f 1064
+    PK testpb.Supply foo -> {"denom":"foo","amount":100}
+GET 010100626f6200666f6f 
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo"}
+GET 010100626f6200666f6f 
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo"}
+ORM INSERT testpb.Balance {"address":"bob","denom":"foo","amount":100}
+SET 010100626f6200666f6f 1864
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo","amount":100}
+SET 010101666f6f00626f62 
+    IDX testpb.Balance denom/address : foo/bob -> bob/foo
+GET 010100626f6200666f6f 1864
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo","amount":100}
+GET 010200666f6f 1064
+    PK testpb.Supply foo -> {"denom":"foo","amount":100}
+GET 010100626f6200666f6f 1864
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo","amount":100}
+GET 010100626f6200666f6f 1864
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo","amount":100}
+ORM UPDATE testpb.Balance {"address":"bob","denom":"foo","amount":100} -> {"address":"bob","denom":"foo","amount":70}
+SET 010100626f6200666f6f 1846
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo","amount":70}
+GET 01010073616c6c7900666f6f 
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo"}
+GET 01010073616c6c7900666f6f 
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo"}
+ORM INSERT testpb.Balance {"address":"sally","denom":"foo","amount":30}
+SET 01010073616c6c7900666f6f 181e
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo","amount":30}
+SET 010101666f6f0073616c6c79 
+    IDX testpb.Balance denom/address : foo/sally -> sally/foo
+GET 010100626f6200666f6f 1846
+    PK testpb.Balance bob/foo -> {"address":"bob","denom":"foo","amount":70}
+GET 01010073616c6c7900666f6f 181e
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo","amount":30}
+GET 010200666f6f 1064
+    PK testpb.Supply foo -> {"denom":"foo","amount":100}
+GET 010200666f6f 1064
+    PK testpb.Supply foo -> {"denom":"foo","amount":100}
+ORM UPDATE testpb.Supply {"denom":"foo","amount":100} -> {"denom":"foo","amount":97}
+SET 010200666f6f 1061
+    PK testpb.Supply foo -> {"denom":"foo","amount":97}
+GET 01010073616c6c7900666f6f 181e
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo","amount":30}
+GET 01010073616c6c7900666f6f 181e
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo","amount":30}
+ORM UPDATE testpb.Balance {"address":"sally","denom":"foo","amount":30} -> {"address":"sally","denom":"foo","amount":27}
+SET 01010073616c6c7900666f6f 181b
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo","amount":27}
+GET 01010073616c6c7900666f6f 181b
+    PK testpb.Balance sally/foo -> {"address":"sally","denom":"foo","amount":27}
+GET 010200666f6f 1061
+    PK testpb.Supply foo -> {"denom":"foo","amount":97}