Update x/gov to use Any (#6147)

* Update x/gov to use Any

* Fixes

* Remove MsgSubmitProposalLegacy

* Update CHANGELOG.md

* Add RegisterInterfaces for x/distribution, x/params, & x/upgrade

* Fix query JSON issue

* Fix gov tests

* Revert custom Any Equals

* Re-remove types

* Rename receivers

* Fix imports in gov

* Sort imports

* Make amino JSON signing work with Any

* Run proto-gen

* Create full amino wrapper

* Fix errors

* Fixes

* Fix tests

* Test fixes

* Fix tests

* Linting

* Update ADR 019 and CHANGELOG

* Updated ADR 019

* Extract Marshal/UnmarshalProposal

* fix error

* lint

* linting

* linting

* Update client/keys/parse.go

Co-authored-by: Marko <marbar3778@yahoo.com>

* linting

* Update docs/architecture/adr-019-protobuf-state-encoding.md

Co-authored-by: Alexander Bezobchuk <alexanderbez@users.noreply.github.com>

* Update docs/architecture/adr-019-protobuf-state-encoding.md

Co-authored-by: Alexander Bezobchuk <alexanderbez@users.noreply.github.com>

* Address review feedback

* Add godocs

* Fix errors

* fix errors

* revert file

* Address review feedback

* Address review feedback

* Stacktrace debug flag

* Fix tests

* Address review feedback

Co-authored-by: sahith-narahari <sahithnarahari@gmail.com>
Co-authored-by: Marko <marbar3778@yahoo.com>
Co-authored-by: Alexander Bezobchuk <alexanderbez@users.noreply.github.com>

codec/amino.go

@@ -8,6 +8,8 @@ import (
 	amino "github.com/tendermint/go-amino"
 	cryptoamino "github.com/tendermint/tendermint/crypto/encoding/amino"
 	tmtypes "github.com/tendermint/tendermint/types"
+
+	"github.com/cosmos/cosmos-sdk/codec/types"
 )
 
 // Cdc defines a global generic sealed Amino codec to be used throughout sdk. It
@@ -17,29 +19,38 @@ import (
 var Cdc *Codec
 
 func init() {
-	cdc := New()
-	RegisterCrypto(cdc)
-	RegisterEvidences(cdc)
-	Cdc = cdc.Seal()
+	Cdc = New()
+	RegisterCrypto(Cdc)
+	RegisterEvidences(Cdc)
+	Cdc.Seal()
 }
 
-// Codec defines a type alias for an Amino codec.
-type Codec = amino.Codec
+// deprecated: Codec defines a wrapper for an Amino codec that properly handles protobuf
+// types with Any's
+type Codec struct {
+	Amino *amino.Codec
+}
+
+var _ JSONMarshaler = &Codec{}
+
+func (cdc *Codec) Seal() {
+	cdc.Amino.Seal()
+}
 
 func New() *Codec {
-	return amino.NewCodec()
+	return &Codec{amino.NewCodec()}
 }
 
 // RegisterCrypto registers all crypto dependency types with the provided Amino
 // codec.
 func RegisterCrypto(cdc *Codec) {
-	cryptoamino.RegisterAmino(cdc)
+	cryptoamino.RegisterAmino(cdc.Amino)
 }
 
 // RegisterEvidences registers Tendermint evidence types with the provided Amino
 // codec.
 func RegisterEvidences(cdc *Codec) {
-	tmtypes.RegisterEvidences(cdc)
+	tmtypes.RegisterEvidences(cdc.Amino)
 }
 
 // MarshalJSONIndent provides a utility for indented JSON encoding of an object
@@ -68,3 +79,132 @@ func MustMarshalJSONIndent(m JSONMarshaler, obj interface{}) []byte {
 
 	return bz
 }
+
+func (cdc *Codec) marshalAnys(o interface{}) error {
+	return types.UnpackInterfaces(o, types.AminoPacker{Cdc: cdc.Amino})
+}
+
+func (cdc *Codec) unmarshalAnys(o interface{}) error {
+	return types.UnpackInterfaces(o, types.AminoUnpacker{Cdc: cdc.Amino})
+}
+
+func (cdc *Codec) jsonMarshalAnys(o interface{}) error {
+	return types.UnpackInterfaces(o, types.AminoJSONPacker{Cdc: cdc.Amino})
+}
+
+func (cdc *Codec) jsonUnmarshalAnys(o interface{}) error {
+	return types.UnpackInterfaces(o, types.AminoJSONUnpacker{Cdc: cdc.Amino})
+}
+
+func (cdc *Codec) MarshalBinaryBare(o interface{}) ([]byte, error) {
+	err := cdc.marshalAnys(o)
+	if err != nil {
+		return nil, err
+	}
+	return cdc.Amino.MarshalBinaryBare(o)
+}
+
+func (cdc *Codec) MustMarshalBinaryBare(o interface{}) []byte {
+	bz, err := cdc.MarshalBinaryBare(o)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (cdc *Codec) MarshalBinaryLengthPrefixed(o interface{}) ([]byte, error) {
+	err := cdc.marshalAnys(o)
+	if err != nil {
+		return nil, err
+	}
+	return cdc.Amino.MarshalBinaryLengthPrefixed(o)
+}
+
+func (cdc *Codec) MustMarshalBinaryLengthPrefixed(o interface{}) []byte {
+	bz, err := cdc.MarshalBinaryLengthPrefixed(o)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (cdc *Codec) UnmarshalBinaryBare(bz []byte, ptr interface{}) error {
+	err := cdc.Amino.UnmarshalBinaryBare(bz, ptr)
+	if err != nil {
+		return err
+	}
+	return cdc.unmarshalAnys(ptr)
+}
+
+func (cdc *Codec) MustUnmarshalBinaryBare(bz []byte, ptr interface{}) {
+	err := cdc.UnmarshalBinaryBare(bz, ptr)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cdc *Codec) UnmarshalBinaryLengthPrefixed(bz []byte, ptr interface{}) error {
+	err := cdc.Amino.UnmarshalBinaryLengthPrefixed(bz, ptr)
+	if err != nil {
+		return err
+	}
+	return cdc.unmarshalAnys(ptr)
+}
+
+func (cdc *Codec) MustUnmarshalBinaryLengthPrefixed(bz []byte, ptr interface{}) {
+	err := cdc.UnmarshalBinaryLengthPrefixed(bz, ptr)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cdc *Codec) MarshalJSON(o interface{}) ([]byte, error) {
+	err := cdc.jsonMarshalAnys(o)
+	if err != nil {
+		return nil, err
+	}
+	return cdc.Amino.MarshalJSON(o)
+}
+
+func (cdc *Codec) MustMarshalJSON(o interface{}) []byte {
+	bz, err := cdc.MarshalJSON(o)
+	if err != nil {
+		panic(err)
+	}
+	return bz
+}
+
+func (cdc *Codec) UnmarshalJSON(bz []byte, ptr interface{}) error {
+	err := cdc.Amino.UnmarshalJSON(bz, ptr)
+	if err != nil {
+		return err
+	}
+	return cdc.jsonUnmarshalAnys(ptr)
+}
+
+func (cdc *Codec) MustUnmarshalJSON(bz []byte, ptr interface{}) {
+	err := cdc.UnmarshalJSON(bz, ptr)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (*Codec) UnpackAny(*types.Any, interface{}) error {
+	return fmt.Errorf("AminoCodec can't handle unpack protobuf Any's")
+}
+
+func (cdc *Codec) RegisterInterface(ptr interface{}, iopts *amino.InterfaceOptions) {
+	cdc.Amino.RegisterInterface(ptr, iopts)
+}
+
+func (cdc *Codec) RegisterConcrete(o interface{}, name string, copts *amino.ConcreteOptions) {
+	cdc.Amino.RegisterConcrete(o, name, copts)
+}
+
+func (cdc *Codec) MarshalJSONIndent(o interface{}, prefix, indent string) ([]byte, error) {
+	err := cdc.jsonMarshalAnys(o)
+	if err != nil {
+		panic(err)
+	}
+	return cdc.Amino.MarshalJSONIndent(o, prefix, indent)
+}

codec/amino_codec.go

@@ -1,133 +1,45 @@
 package codec
 
-import (
-	"fmt"
-
-	"github.com/cosmos/cosmos-sdk/codec/types"
-)
-
-// AminoCodec defines a codec that utilizes Amino for both binary and JSON
+// AminoCodec defines a codec that utilizes Codec for both binary and JSON
 // encoding.
 type AminoCodec struct {
-	amino *Codec
+	*Codec
 }
 
-func NewAminoCodec(amino *Codec) Marshaler {
-	return &AminoCodec{amino}
-}
+var _ Marshaler = &AminoCodec{}
 
-func (ac *AminoCodec) marshalAnys(o ProtoMarshaler) error {
-	return types.UnpackInterfaces(o, types.AminoPacker{Cdc: ac.amino})
-}
-
-func (ac *AminoCodec) unmarshalAnys(o ProtoMarshaler) error {
-	return types.UnpackInterfaces(o, types.AminoUnpacker{Cdc: ac.amino})
-}
-
-func (ac *AminoCodec) jsonMarshalAnys(o interface{}) error {
-	return types.UnpackInterfaces(o, types.AminoJSONPacker{Cdc: ac.amino})
-}
-
-func (ac *AminoCodec) jsonUnmarshalAnys(o interface{}) error {
-	return types.UnpackInterfaces(o, types.AminoJSONUnpacker{Cdc: ac.amino})
+func NewAminoCodec(codec *Codec) *AminoCodec {
+	return &AminoCodec{Codec: codec}
 }
 
 func (ac *AminoCodec) MarshalBinaryBare(o ProtoMarshaler) ([]byte, error) {
-	err := ac.marshalAnys(o)
-	if err != nil {
-		return nil, err
-	}
-	return ac.amino.MarshalBinaryBare(o)
+	return ac.Codec.MarshalBinaryBare(o)
 }
 
 func (ac *AminoCodec) MustMarshalBinaryBare(o ProtoMarshaler) []byte {
-	err := ac.marshalAnys(o)
-	if err != nil {
-		panic(err)
-	}
-	return ac.amino.MustMarshalBinaryBare(o)
+	return ac.Codec.MustMarshalBinaryBare(o)
 }
 
 func (ac *AminoCodec) MarshalBinaryLengthPrefixed(o ProtoMarshaler) ([]byte, error) {
-	err := ac.marshalAnys(o)
-	if err != nil {
-		return nil, err
-	}
-	return ac.amino.MarshalBinaryLengthPrefixed(o)
+	return ac.Codec.MarshalBinaryLengthPrefixed(o)
 }
 
 func (ac *AminoCodec) MustMarshalBinaryLengthPrefixed(o ProtoMarshaler) []byte {
-	err := ac.marshalAnys(o)
-	if err != nil {
-		panic(err)
-	}
-	return ac.amino.MustMarshalBinaryLengthPrefixed(o)
+	return ac.Codec.MustMarshalBinaryLengthPrefixed(o)
 }
 
 func (ac *AminoCodec) UnmarshalBinaryBare(bz []byte, ptr ProtoMarshaler) error {
-	err := ac.amino.UnmarshalBinaryBare(bz, ptr)
-	if err != nil {
-		return err
-	}
-	return ac.unmarshalAnys(ptr)
+	return ac.Codec.UnmarshalBinaryBare(bz, ptr)
 }
 
 func (ac *AminoCodec) MustUnmarshalBinaryBare(bz []byte, ptr ProtoMarshaler) {
-	ac.amino.MustUnmarshalBinaryBare(bz, ptr)
-	err := ac.unmarshalAnys(ptr)
-	if err != nil {
-		panic(err)
-	}
+	ac.Codec.MustUnmarshalBinaryBare(bz, ptr)
 }
 
 func (ac *AminoCodec) UnmarshalBinaryLengthPrefixed(bz []byte, ptr ProtoMarshaler) error {
-	err := ac.amino.UnmarshalBinaryLengthPrefixed(bz, ptr)
-	if err != nil {
-		return err
-	}
-	return ac.unmarshalAnys(ptr)
+	return ac.Codec.UnmarshalBinaryLengthPrefixed(bz, ptr)
 }
 
 func (ac *AminoCodec) MustUnmarshalBinaryLengthPrefixed(bz []byte, ptr ProtoMarshaler) {
-	ac.amino.MustUnmarshalBinaryLengthPrefixed(bz, ptr)
-	err := ac.unmarshalAnys(ptr)
-	if err != nil {
-		panic(err)
-	}
-}
-
-func (ac *AminoCodec) MarshalJSON(o interface{}) ([]byte, error) {
-	err := ac.jsonMarshalAnys(o)
-	if err != nil {
-		return nil, err
-	}
-	return ac.amino.MarshalJSON(o)
-}
-
-func (ac *AminoCodec) MustMarshalJSON(o interface{}) []byte {
-	err := ac.jsonMarshalAnys(o)
-	if err != nil {
-		panic(err)
-	}
-	return ac.amino.MustMarshalJSON(o)
-}
-
-func (ac *AminoCodec) UnmarshalJSON(bz []byte, ptr interface{}) error {
-	err := ac.amino.UnmarshalJSON(bz, ptr)
-	if err != nil {
-		return err
-	}
-	return ac.jsonUnmarshalAnys(ptr)
-}
-
-func (ac *AminoCodec) MustUnmarshalJSON(bz []byte, ptr interface{}) {
-	ac.amino.MustUnmarshalJSON(bz, ptr)
-	err := ac.jsonUnmarshalAnys(ptr)
-	if err != nil {
-		panic(err)
-	}
-}
-
-func (*AminoCodec) UnpackAny(*types.Any, interface{}) error {
-	return fmt.Errorf("AminoCodec can't handle unpack protobuf Any's")
+	ac.Codec.MustUnmarshalBinaryLengthPrefixed(bz, ptr)
 }

codec/amino_codec_test.go

@@ -6,14 +6,13 @@ import (
 	"github.com/cosmos/cosmos-sdk/codec/types"
 
 	"github.com/stretchr/testify/require"
-	amino "github.com/tendermint/go-amino"
 
 	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/codec/testdata"
 )
 
-func createTestCodec() *amino.Codec {
-	cdc := amino.NewCodec()
+func createTestCodec() *codec.Codec {
+	cdc := codec.New()
 
 	cdc.RegisterInterface((*testdata.Animal)(nil), nil)
 	cdc.RegisterConcrete(testdata.Dog{}, "testdata/Dog", nil)

codec/types/amino_compat.go

@@ -3,6 +3,9 @@ package types
 import (
 	"fmt"
 	"reflect"
+	"runtime/debug"
+
+	"github.com/gogo/protobuf/proto"
 
 	amino "github.com/tendermint/go-amino"
 )
@@ -13,10 +16,27 @@ type aminoCompat struct {
 	err    error
 }
 
+var Debug = false
+
+func aminoCompatError(errType string, x interface{}) error {
+	if Debug {
+		debug.PrintStack()
+	}
+	return fmt.Errorf(
+		"amino %s Any marshaling error for %+v, this is likely because "+
+			"amino is being used directly (instead of codec.Codec which is preferred) "+
+			"or UnpackInterfacesMessage is not defined for some type which contains "+
+			"a protobuf Any either directly or via one of its members. To see a "+
+			"stacktrace of where the error is coming from, set the var Debug = true "+
+			"in codec/types/amino_compat.go",
+		errType, x,
+	)
+}
+
 func (any Any) MarshalAmino() ([]byte, error) {
 	ac := any.aminoCompat
 	if ac == nil {
-		return nil, fmt.Errorf("can't amino unmarshal")
+		return nil, aminoCompatError("binary unmarshal", any)
 	}
 	return ac.bz, ac.err
 }
@@ -32,7 +52,7 @@ func (any *Any) UnmarshalAmino(bz []byte) error {
 func (any Any) MarshalJSON() ([]byte, error) {
 	ac := any.aminoCompat
 	if ac == nil {
-		return nil, fmt.Errorf("can't JSON marshal")
+		return nil, aminoCompatError("JSON marshal", any)
 	}
 	return ac.jsonBz, ac.err
 }
@@ -56,7 +76,7 @@ var _ AnyUnpacker = AminoUnpacker{}
 func (a AminoUnpacker) UnpackAny(any *Any, iface interface{}) error {
 	ac := any.aminoCompat
 	if ac == nil {
-		return fmt.Errorf("can't amino unmarshal %T", iface)
+		return aminoCompatError("binary unmarshal", reflect.TypeOf(iface))
 	}
 	err := a.Cdc.UnmarshalBinaryBare(ac.bz, iface)
 	if err != nil {
@@ -67,7 +87,19 @@ func (a AminoUnpacker) UnpackAny(any *Any, iface interface{}) error {
 	if err != nil {
 		return err
 	}
-	any.cachedValue = val
+	if m, ok := val.(proto.Message); ok {
+		err := any.Pack(m)
+		if err != nil {
+			return err
+		}
+	} else {
+		any.cachedValue = val
+	}
+
+	// this is necessary for tests that use reflect.DeepEqual and compare
+	// proto vs amino marshaled values
+	any.aminoCompat = nil
+
 	return nil
 }
 
@@ -103,7 +135,7 @@ var _ AnyUnpacker = AminoJSONUnpacker{}
 func (a AminoJSONUnpacker) UnpackAny(any *Any, iface interface{}) error {
 	ac := any.aminoCompat
 	if ac == nil {
-		return fmt.Errorf("can't amino unmarshal %T", iface)
+		return aminoCompatError("JSON unmarshal", reflect.TypeOf(iface))
 	}
 	err := a.Cdc.UnmarshalJSON(ac.jsonBz, iface)
 	if err != nil {
@@ -114,7 +146,19 @@ func (a AminoJSONUnpacker) UnpackAny(any *Any, iface interface{}) error {
 	if err != nil {
 		return err
 	}
-	any.cachedValue = val
+	if m, ok := val.(proto.Message); ok {
+		err := any.Pack(m)
+		if err != nil {
+			return err
+		}
+	} else {
+		any.cachedValue = val
+	}
+
+	// this is necessary for tests that use reflect.DeepEqual and compare
+	// proto vs amino marshaled values
+	any.aminoCompat = nil
+
 	return nil
 }
 

codec/types/interface_registry.go

@@ -112,6 +112,11 @@ func (registry *interfaceRegistry) RegisterImplementations(iface interface{}, im
 }
 
 func (registry *interfaceRegistry) UnpackAny(any *Any, iface interface{}) error {
+	if any.TypeUrl == "" {
+		// if TypeUrl is empty return nil because without it we can't actually unpack anything
+		return nil
+	}
+
 	rv := reflect.ValueOf(iface)
 	if rv.Kind() != reflect.Ptr {
 		return fmt.Errorf("UnpackAny expects a pointer")