feat(orm): ModuleDB JSON import/export/default/validate methods (#11101)

* feat(orm): ModuleDB JSON methods

* WIP

* WIP on JSON

* WIP

* WIP

* tests and docs

* revert

* tests and docs

* docs

* address review comments

orm/model/kv/store.go

@@ -1,50 +0,0 @@
-// Package kvstore defines the abstract interfaces which ORM tables and indexes
-// use for reading and writing data against a KV-store backend.
-package kv
-
-import (
-	dbm "github.com/tendermint/tm-db"
-)
-
-// ReadonlyStore is an interface for readonly access to a kv-store.
-type ReadonlyStore interface {
-	// Get fetches the value of the given key, or nil if it does not exist.
-	// CONTRACT: key, value readonly []byte
-	Get(key []byte) ([]byte, error)
-
-	// Has checks if a key exists.
-	// CONTRACT: key, value readonly []byte
-	Has(key []byte) (bool, error)
-
-	// Iterator returns an iterator over a domain of keys, in ascending order. The caller must call
-	// Close when done. End is exclusive, and start must be less than end. A nil start iterates
-	// from the first key, and a nil end iterates to the last key (inclusive). Empty keys are not
-	// valid.
-	// CONTRACT: No writes may happen within a domain while an iterator exists over it.
-	// CONTRACT: start, end readonly []byte
-	Iterator(start, end []byte) (Iterator, error)
-
-	// ReverseIterator returns an iterator over a domain of keys, in descending order. The caller
-	// must call Close when done. End is exclusive, and start must be less than end. A nil end
-	// iterates from the last key (inclusive), and a nil start iterates to the first key (inclusive).
-	// Empty keys are not valid.
-	// CONTRACT: No writes may happen within a domain while an iterator exists over it.
-	// CONTRACT: start, end readonly []byte
-	ReverseIterator(start, end []byte) (Iterator, error)
-}
-
-// Iterator aliases github.com/tendermint/tm-db.Iterator.
-type Iterator = dbm.Iterator
-
-// Store is an interface for writing to a kv-store.
-type Store interface {
-	ReadonlyStore
-
-	// Set sets the value for the given key, replacing it if it already exists.
-	// CONTRACT: key, value readonly []byte
-	Set(key, value []byte) error
-
-	// Delete deletes the key, or does nothing if the key does not exist.
-	// CONTRACT: key readonly []byte
-	Delete(key []byte) error
-}

orm/model/ormdb/json.go

@@ -0,0 +1,119 @@
+package ormdb
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormjson"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	"github.com/cosmos/cosmos-sdk/errors"
+)
+
+func (m moduleDB) DefaultJSON(target ormjson.WriteTarget) error {
+	for name, table := range m.tablesByName {
+		w, err := target.OpenWriter(name)
+		if err != nil {
+			return err
+		}
+
+		_, err = w.Write(table.DefaultJSON())
+		if err != nil {
+			return err
+		}
+
+		err = w.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m moduleDB) ValidateJSON(source ormjson.ReadSource) error {
+	errMap := map[protoreflect.FullName]error{}
+	for name, table := range m.tablesByName {
+		r, err := source.OpenReader(name)
+		if err != nil {
+			return err
+		}
+
+		err = table.ValidateJSON(r)
+		if err != nil {
+			errMap[name] = err
+		}
+
+		err = r.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(errMap) != 0 {
+		var allErrors string
+		for name, err := range errMap {
+			allErrors += fmt.Sprintf("Error in JSON for table %s: %v\n", name, err)
+		}
+		return ormerrors.JSONValidationError.Wrap(allErrors)
+	}
+
+	return nil
+}
+
+func (m moduleDB) ImportJSON(ctx context.Context, source ormjson.ReadSource) error {
+	var names []string
+	for name := range m.tablesByName {
+		names = append(names, string(name))
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		fullName := protoreflect.FullName(name)
+		table := m.tablesByName[fullName]
+
+		r, err := source.OpenReader(fullName)
+		if err != nil {
+			return errors.Wrapf(err, "table %s", fullName)
+		}
+
+		if r == nil {
+			continue
+		}
+
+		err = table.ImportJSON(ctx, r)
+		if err != nil {
+			return errors.Wrapf(err, "table %s", fullName)
+		}
+
+		err = r.Close()
+		if err != nil {
+			return errors.Wrapf(err, "table %s", fullName)
+		}
+	}
+
+	return nil
+}
+
+func (m moduleDB) ExportJSON(ctx context.Context, sink ormjson.WriteTarget) error {
+	for name, table := range m.tablesByName {
+		w, err := sink.OpenWriter(name)
+		if err != nil {
+			return err
+		}
+
+		err = table.ExportJSON(ctx, w)
+		if err != nil {
+			return err
+		}
+
+		err = w.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

orm/model/ormdb/module.go

@@ -6,6 +6,8 @@ import (
 	"encoding/binary"
 	"math"
 
+	"github.com/cosmos/cosmos-sdk/orm/types/ormjson"
+
 	"google.golang.org/protobuf/reflect/protodesc"
 
 	"github.com/cosmos/cosmos-sdk/orm/encoding/encodeutil"
@@ -31,7 +33,22 @@ type ModuleSchema struct {
 }
 
 // ModuleDB defines the ORM database type to be used by modules.
-type ModuleDB = ormtable.Schema
+type ModuleDB interface {
+	ormtable.Schema
+
+	// DefaultJSON writes default JSON for each table in the module to the target.
+	DefaultJSON(ormjson.WriteTarget) error
+
+	// ValidateJSON validates JSON for each table in the module.
+	ValidateJSON(ormjson.ReadSource) error
+
+	// ImportJSON imports JSON for each table in the module which has JSON
+	// defined in the read source.
+	ImportJSON(context.Context, ormjson.ReadSource) error
+
+	// ExportJSON exports JSON for each table in the module.
+	ExportJSON(context.Context, ormjson.WriteTarget) error
+}
 
 type moduleDB struct {
 	prefix       []byte

orm/model/ormdb/module_test.go

@@ -3,10 +3,17 @@ package ormdb_test
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"testing"
 
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+
+	"github.com/cosmos/cosmos-sdk/orm/testing/ormtest"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormjson"
+
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/golden"
@@ -157,14 +164,14 @@ func TestModuleDB(t *testing.T) {
 	db, err := ormdb.NewModuleDB(TestBankSchema, ormdb.ModuleDBOptions{})
 	assert.NilError(t, err)
 	debugBuf := &strings.Builder{}
-	store := testkv.NewDebugBackend(
-		testkv.NewSharedMemBackend(),
+	backend := ormtest.NewMemoryBackend()
+	ctx := ormtable.WrapContextDefault(testkv.NewDebugBackend(
+		backend,
 		&testkv.EntryCodecDebugger{
 			EntryCodec: db,
 			Print:      func(s string) { debugBuf.WriteString(s + "\n") },
 		},
-	)
-	ctx := ormtable.WrapContextDefault(store)
+	))
 
 	// create keeper
 	k, err := newKeeper(db)
@@ -205,7 +212,7 @@ func TestModuleDB(t *testing.T) {
 	golden.Assert(t, debugBuf.String(), "bank_scenario.golden")
 
 	// check decode & encode
-	it, err := store.CommitmentStore().Iterator(nil, nil)
+	it, err := backend.CommitmentStore().Iterator(nil, nil)
 	assert.NilError(t, err)
 	for it.Valid() {
 		entry, err := db.DecodeEntry(it.Key(), it.Value())
@@ -216,4 +223,33 @@ func TestModuleDB(t *testing.T) {
 		assert.Assert(t, bytes.Equal(v, it.Value()))
 		it.Next()
 	}
+
+	// check JSON
+	target := ormjson.NewRawMessageTarget()
+	assert.NilError(t, db.DefaultJSON(target))
+	rawJson, err := target.JSON()
+	assert.NilError(t, err)
+	golden.Assert(t, string(rawJson), "default_json.golden")
+
+	target = ormjson.NewRawMessageTarget()
+	assert.NilError(t, db.ExportJSON(ctx, target))
+	rawJson, err = target.JSON()
+	assert.NilError(t, err)
+
+	badJSON := `{
+  "testpb.Balance": 5,
+  "testpb.Supply": {}
+}
+`
+	source, err := ormjson.NewRawMessageSource(json.RawMessage(badJSON))
+	assert.NilError(t, err)
+	assert.ErrorIs(t, db.ValidateJSON(source), ormerrors.JSONValidationError)
+
+	backend2 := ormtest.NewMemoryBackend()
+	ctx2 := ormtable.WrapContextDefault(backend2)
+	source, err = ormjson.NewRawMessageSource(rawJson)
+	assert.NilError(t, err)
+	assert.NilError(t, db.ValidateJSON(source))
+	assert.NilError(t, db.ImportJSON(ctx2, source))
+	testkv.AssertBackendsEqual(t, backend, backend2)
 }

orm/model/ormdb/testdata/default_json.golden

@@ -0,0 +1,4 @@
+{
+  "testpb.Balance": [],
+  "testpb.Supply": []
+}

orm/model/ormtable/auto_increment.go

@@ -10,7 +10,7 @@ import (
 	"google.golang.org/protobuf/reflect/protoreflect"
 
 	"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
 	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
 )
 
@@ -125,7 +125,7 @@ func (t autoIncrementTable) ValidateJSON(reader io.Reader) error {
 		messageRef := message.ProtoReflect()
 		id := messageRef.Get(t.autoIncField).Uint()
 		if id > maxID {
-			return fmt.Errorf("invalid ID %d, expected a value <= %d", id, maxID)
+			return fmt.Errorf("invalid ID %d, expected a value <= %d, the highest sequence number", id, maxID)
 		}
 
 		if t.customJSONValidator != nil {
@@ -152,7 +152,7 @@ func (t autoIncrementTable) ImportJSON(ctx context.Context, reader io.Reader) er
 			return err
 		} else {
 			if id > maxID {
-				return fmt.Errorf("invalid ID %d, expected a value <= %d", id, maxID)
+				return fmt.Errorf("invalid ID %d, expected a value <= %d, the highest sequence number", id, maxID)
 			}
 			// we do have an ID and calling Save will fail because it expects
 			// either no ID or SAVE_MODE_UPDATE. So instead we drop one level
@@ -213,18 +213,20 @@ func (t autoIncrementTable) ExportJSON(ctx context.Context, writer io.Writer) er
 		return err
 	}
 
-	bz, err := json.Marshal(seq)
-	if err != nil {
-		return err
-	}
-	_, err = writer.Write(bz)
-	if err != nil {
-		return err
-	}
+	if seq != 0 {
+		bz, err := json.Marshal(seq)
+		if err != nil {
+			return err
+		}
+		_, err = writer.Write(bz)
+		if err != nil {
+			return err
+		}
 
-	_, err = writer.Write([]byte(",\n"))
-	if err != nil {
-		return err
+		_, err = writer.Write([]byte(",\n"))
+		if err != nil {
+			return err
+		}
 	}
 
 	return t.doExportJSON(ctx, writer)

orm/model/ormtable/backend.go

@@ -3,7 +3,7 @@ package ormtable
 import (
 	"context"
 
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
 )
 
 // ReadBackend defines the type used for read-only ORM operations.

orm/model/ormtable/batch.go

@@ -1,6 +1,8 @@
 package ormtable
 
-import "github.com/cosmos/cosmos-sdk/orm/model/kv"
+import (
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
+)
 
 type batchIndexCommitmentWriter struct {
 	Backend

orm/model/ormtable/index.go

@@ -3,11 +3,12 @@ package ormtable
 import (
 	"context"
 
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
+
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
 	"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
 	"github.com/cosmos/cosmos-sdk/orm/model/ormlist"
 )
 

orm/model/ormtable/index_impl.go

@@ -3,9 +3,10 @@ package ormtable
 import (
 	"context"
 
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
+
 	"github.com/cosmos/cosmos-sdk/orm/internal/fieldnames"
 
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
 	"github.com/cosmos/cosmos-sdk/orm/model/ormlist"
 
 	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"

orm/model/ormtable/iterator.go

@@ -4,14 +4,12 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
-	"github.com/cosmos/cosmos-sdk/orm/encoding/encodeutil"
-
 	queryv1beta1 "github.com/cosmos/cosmos-sdk/api/cosmos/base/query/v1beta1"
-
+	"github.com/cosmos/cosmos-sdk/orm/encoding/encodeutil"
 	"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
 	"github.com/cosmos/cosmos-sdk/orm/internal/listinternal"
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
 	"github.com/cosmos/cosmos-sdk/orm/model/ormlist"
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
 )
 
 // Iterator defines the interface for iterating over indexes.

orm/model/ormtable/table_test.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
+
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/testing/protocmp"
@@ -21,7 +23,6 @@ import (
 	"github.com/cosmos/cosmos-sdk/orm/internal/testkv"
 	"github.com/cosmos/cosmos-sdk/orm/internal/testpb"
 	"github.com/cosmos/cosmos-sdk/orm/internal/testutil"
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
 	"github.com/cosmos/cosmos-sdk/orm/model/ormlist"
 	"github.com/cosmos/cosmos-sdk/orm/model/ormtable"
 	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"

orm/model/ormtable/unique.go

@@ -3,9 +3,10 @@ package ormtable
 import (
 	"context"
 
+	"github.com/cosmos/cosmos-sdk/orm/types/kv"
+
 	"github.com/cosmos/cosmos-sdk/orm/internal/fieldnames"
 
-	"github.com/cosmos/cosmos-sdk/orm/model/kv"
 	"github.com/cosmos/cosmos-sdk/orm/model/ormlist"
 
 	"github.com/cosmos/cosmos-sdk/orm/encoding/encodeutil"