LaconicNetwork/cosmos-sdk
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

refactor(x/genutil,server): add export functions to x/gentutil (#18303)

Browse Source
This commit is contained in:
Julien Robert 2023-10-30 16:49:49 +01:00 committed by GitHub
parent 393de266c8
commit d82503e24e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 78 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -79,6 +79,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
### API Breaking Changes
* (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `x/genutil` now handles the application export. `server.AddCommands` does not take an `AppExporter` but instead `genutilcli.Commands` does.
* (x/gov) [#18173](https://github.com/cosmos/cosmos-sdk/pull/18173) Gov Hooks now returns error and are "blocking" if they fail. Expect for `AfterProposalFailedMinDeposit` and `AfterProposalVotingPeriodEnded` that will log the error and continue.
* (x/gov/testutil) [#17986](https://github.com/cosmos/cosmos-sdk/pull/18036) `MsgDeposit` has been removed because of AutoCLI migration.
* (x/staking/testutil) [#17986](https://github.com/cosmos/cosmos-sdk/pull/17986) `MsgRedelegateExec`, `MsgUnbondExec` has been removed because of AutoCLI migration.
@ -187,6 +188,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
### CLI Breaking Changes
* (server) [#18303](https://github.com/cosmos/cosmos-sdk/pull/18303) `appd export` has moved with other genesis commands, use `appd genesis export` instead.
* (x/auth/vesting) [#18100](https://github.com/cosmos/cosmos-sdk/pull/18100) `appd tx vesting create-vesting-account` takes an amount of coin as last argument instead of second. Coins are space separated.
* (x/distribution) [#17963](https://github.com/cosmos/cosmos-sdk/pull/17963) `appd tx distribution withdraw-rewards` now only withdraws rewards for the delegator's own delegations. For withdrawing validators commission, use `appd tx distribution withdraw-validator-commission`.

6
docs/architecture/adr-041-in-place-store-migrations.md
View File

@ -14,7 +14,7 @@ This ADR introduces a mechanism to perform in-place state store migrations durin
## Context
When a chain upgrade introduces state-breaking changes inside modules, the current procedure consists of exporting the whole state into a JSON file (via the `simd export` command), running migration scripts on the JSON file (`simd genesis migrate` command), clearing the stores (`simd unsafe-reset-all` command), and starting a new chain with the migrated JSON file as new genesis (optionally with a custom initial block height). An example of such a procedure can be seen [in the Cosmos Hub 3->4 migration guide](https://github.com/cosmos/gaia/blob/v4.0.3/docs/migration/cosmoshub-3.md#upgrade-procedure).
When a chain upgrade introduces state-breaking changes inside modules, the current procedure consists of exporting the whole state into a JSON file (via the `simd genesis export` command), running migration scripts on the JSON file (`simd genesis migrate` command), clearing the stores (`simd unsafe-reset-all` command), and starting a new chain with the migrated JSON file as new genesis (optionally with a custom initial block height). An example of such a procedure can be seen [in the Cosmos Hub 3->4 migration guide](https://github.com/cosmos/gaia/blob/v4.0.3/docs/migration/cosmoshub-3.md#upgrade-procedure).
This procedure is cumbersome for multiple reasons:
@ -147,7 +147,7 @@ While modules MUST register their migration functions when bumping ConsensusVers
### Positive
* Perform chain upgrades without manipulating JSON files.
* While no benchmark has been made yet, it is probable that in-place store migrations will take less time than JSON migrations. The main reason supporting this claim is that both the `simd export` command on the old binary and the `InitChain` function on the new binary will be skipped.
* While no benchmark has been made yet, it is probable that in-place store migrations will take less time than JSON migrations. The main reason supporting this claim is that both the `simd genesis export` command on the old binary and the `InitChain` function on the new binary will be skipped.
### Negative
@ -155,7 +155,7 @@ While modules MUST register their migration functions when bumping ConsensusVers
### Neutral
* The Cosmos SDK will continue to support JSON migrations via the existing `simd export` and `simd genesis migrate` commands.
* The Cosmos SDK will continue to support JSON migrations via the existing `simd genesis export` and `simd genesis migrate` commands.
* The current ADR does not allow creating, renaming or deleting stores, only modifying existing store keys and values. The Cosmos SDK already has the `StoreLoader` for those operations.
## Further Discussions

2
server/cmt_cmds.go
View File

@ -372,7 +372,7 @@ func BootstrapStateCmd(appCreator types.AppCreator) *cobra.Command {
}
if height == 0 {
home := serverCtx.Viper.GetString(flags.FlagHome)
db, err := openDB(home, GetAppDBBackend(serverCtx.Viper))
db, err := OpenDB(home, GetAppDBBackend(serverCtx.Viper))
if err != nil {
return err
}

4
server/constructors_test.go
View File

@ -8,9 +8,9 @@ import (
"github.com/stretchr/testify/require"
)
func Test_openDB(t *testing.T) {
func Test_OpenDB(t *testing.T) {
t.Parallel()
_, err := openDB(t.TempDir(), dbm.GoLevelDBBackend)
_, err := OpenDB(t.TempDir(), dbm.GoLevelDBBackend)
require.NoError(t, err)
}

2
server/rollback.go
View File

@ -27,7 +27,7 @@ application.
RunE: func(cmd *cobra.Command, args []string) error {
ctx := GetServerContextFromCmd(cmd)
db, err := openDB(ctx.Config.RootDir, GetAppDBBackend(ctx.Viper))
db, err := OpenDB(ctx.Config.RootDir, GetAppDBBackend(ctx.Viper))
if err != nil {
return err
}

13
server/start.go
View File

@ -27,6 +27,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"cosmossdk.io/log"
pruningtypes "cosmossdk.io/store/pruning/types"
"github.com/cosmos/cosmos-sdk/client"
@ -113,7 +114,7 @@ func StartCmd(appCreator types.AppCreator) *cobra.Command {
// CometBFT.
func StartCmdWithOptions(appCreator types.AppCreator, opts StartCmdOptions) *cobra.Command {
if opts.DBOpener == nil {
opts.DBOpener = openDB
opts.DBOpener = OpenDB
}
cmd := &cobra.Command{
@ -437,7 +438,7 @@ func getAndValidateConfig(svrCtx *Context) (serverconfig.Config, error) {
return config, nil
}
// returns a function which returns the genesis doc from the genesis file.
// getGenDocProvider returns a function which returns the genesis doc from the genesis file.
func getGenDocProvider(cfg *cmtcfg.Config) func() (*cmttypes.GenesisDoc, error) {
return func() (*cmttypes.GenesisDoc, error) {
appGenesis, err := genutiltypes.AppGenesisFromFile(cfg.GenesisFile())
@ -449,11 +450,11 @@ func getGenDocProvider(cfg *cmtcfg.Config) func() (*cmttypes.GenesisDoc, error)
}
}
func setupTraceWriter(svrCtx *Context) (traceWriter io.WriteCloser, cleanup func(), err error) {
// SetupTraceWriter sets up the trace writer and returns a cleanup function.
func SetupTraceWriter(logger log.Logger, traceWriterFile string) (traceWriter io.WriteCloser, cleanup func(), err error) {
// clean up the traceWriter when the server is shutting down
cleanup = func() {}
traceWriterFile := svrCtx.Viper.GetString(flagTraceStore)
traceWriter, err = openTraceWriter(traceWriterFile)
if err != nil {
return traceWriter, cleanup, err
@ -463,7 +464,7 @@ func setupTraceWriter(svrCtx *Context) (traceWriter io.WriteCloser, cleanup func
if traceWriter != nil {
cleanup = func() {
if err = traceWriter.Close(); err != nil {
svrCtx.Logger.Error("failed to close trace writer", "err", err)
logger.Error("failed to close trace writer", "err", err)
}
}
}
@ -626,7 +627,7 @@ func getCtx(svrCtx *Context, block bool) (*errgroup.Group, context.Context) {
}
func startApp(svrCtx *Context, appCreator types.AppCreator, opts StartCmdOptions) (app types.Application, cleanupFn func(), err error) {
traceWriter, traceCleanupFn, err := setupTraceWriter(svrCtx)
traceWriter, traceCleanupFn, err := SetupTraceWriter(svrCtx.Logger, svrCtx.Viper.GetString(flagTraceStore))
if err != nil {
return app, traceCleanupFn, err
}

6
server/util.go
View File

@ -321,7 +321,7 @@ func interceptConfigs(rootViper *viper.Viper, customAppTemplate string, customCo
}
// add server commands
func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, appExport types.AppExporter, addStartFlags types.ModuleInitFlags) {
func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, addStartFlags types.ModuleInitFlags) {
cometCmd := &cobra.Command{
Use: "comet",
Aliases: []string{"cometbft", "tendermint"},
@ -344,7 +344,6 @@ func AddCommands(rootCmd *cobra.Command, appCreator types.AppCreator, appExport
rootCmd.AddCommand(
startCmd,
cometCmd,
ExportCmd(appExport),
version.NewVersionCommand(),
NewRollbackCmd(appCreator),
)
@ -452,7 +451,8 @@ func addrToIP(addr net.Addr) net.IP {
return ip
}
func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
// OpenDB opens the application database using the appropriate driver.
func OpenDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
dataDir := filepath.Join(rootDir, "data")
return dbm.NewDB("application", backendType, dataDir)
}

8
simapp/simd/cmd/commands.go
View File

@ -51,12 +51,12 @@ func initRootCmd(
snapshot.Cmd(newApp),
)
server.AddCommands(rootCmd, newApp, appExport, addModuleInitFlags)
server.AddCommands(rootCmd, newApp, addModuleInitFlags)
// add keybase, auxiliary RPC, query, genesis, and tx child commands
rootCmd.AddCommand(
server.StatusCommand(),
genesisCommand(txConfig, basicManager),
genesisCommand(txConfig, basicManager, appExport),
queryCommand(),
txCommand(),
keys.Commands(),
@ -68,8 +68,8 @@ func addModuleInitFlags(startCmd *cobra.Command) {
}
// genesisCommand builds genesis-related `simd genesis` command. Users may provide application specific commands as a parameter
func genesisCommand(txConfig client.TxConfig, basicManager module.BasicManager, cmds ...*cobra.Command) *cobra.Command {
cmd := genutilcli.Commands(txConfig, basicManager)
func genesisCommand(txConfig client.TxConfig, basicManager module.BasicManager, appExport servertypes.AppExporter, cmds ...*cobra.Command) *cobra.Command {
cmd := genutilcli.Commands(txConfig, basicManager, appExport)
for _, subCmd := range cmds {
cmd.AddCommand(subCmd)

9
tests/e2e/server/export_test.go → tests/e2e/genutil/export_test.go
View File

@ -1,7 +1,7 @@
//go:build e2e
// +build e2e
package server_test
package genutil_test
import (
"bytes"
@ -27,6 +27,7 @@ import (
"github.com/cosmos/cosmos-sdk/server/types"
simtestutil "github.com/cosmos/cosmos-sdk/testutil/sims"
"github.com/cosmos/cosmos-sdk/x/genutil"
genutilcli "github.com/cosmos/cosmos-sdk/x/genutil/client/cli"
genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
)
@ -76,14 +77,14 @@ func TestExportCmd_Height(t *testing.T) {
{
"should export correct height with --height",
[]string{
fmt.Sprintf("--%s=%d", server.FlagHeight, 3),
fmt.Sprintf("--height=%d", 3),
},
5, 4,
},
{
"should export height 0 with --for-zero-height",
[]string{
fmt.Sprintf("--%s=%s", server.FlagForZeroHeight, "true"),
fmt.Sprintf("--for-zero-height=%s", "true"),
},
2, 0,
},
@ -196,7 +197,7 @@ func setupApp(t *testing.T, tempDir string) (*simapp.SimApp, context.Context, ge
_, err = app.Commit()
assert.NilError(t, err)
cmd := server.ExportCmd(func(_ log.Logger, _ dbm.DB, _ io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string, appOptions types.AppOptions, modulesToExport []string) (types.ExportedApp, error) {
cmd := genutilcli.ExportCmd(func(_ log.Logger, _ dbm.DB, _ io.Writer, height int64, forZeroHeight bool, jailAllowedAddrs []string, appOptions types.AppOptions, modulesToExport []string) (types.ExportedApp, error) {
var simApp *simapp.SimApp
if height != -1 {
simApp = simapp.NewSimApp(logger, db, nil, false, appOptions)

16
x/genutil/README.md
View File

@ -12,6 +12,7 @@ The `genutil` package contains a variety of genesis utility functionalities for
* Genesis file migration
* CometBFT related initialization
* Translation of an app genesis to a CometBFT genesis
* Application state export into a genesis file
## Genesis
@ -87,3 +88,18 @@ simd genesis validate-genesis
:::warning
Validate genesis only validates if the genesis is valid at the **current application binary**. For validating a genesis from a previous version of the application, use the `migrate` command to migrate the genesis to the current version.
:::
#### export
Export state to genesis file.
```shell
simd genesis export
```
Some flags are available to customize the export:
* `--for-zero-height`: export the genesis file for a chain with zero height
* `--height [height]`: export the genesis file for a chain with a given height
Read the help for more information.

8
x/genutil/client/cli/commands.go
View File

@ -6,19 +6,20 @@ import (
banktypes "cosmossdk.io/x/bank/types"
"github.com/cosmos/cosmos-sdk/client"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/types/module"
"github.com/cosmos/cosmos-sdk/x/genutil"
genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
)
// Commands adds core sdk's sub-commands into genesis command.
func Commands(txConfig client.TxConfig, moduleBasics module.BasicManager) *cobra.Command {
return CommandsWithCustomMigrationMap(txConfig, moduleBasics, MigrationMap)
func Commands(txConfig client.TxConfig, moduleBasics module.BasicManager, appExport servertypes.AppExporter) *cobra.Command {
return CommandsWithCustomMigrationMap(txConfig, moduleBasics, appExport, MigrationMap)
}
// CommandsWithCustomMigrationMap adds core sdk's sub-commands into genesis command with custom migration map.
// This custom migration map can be used by the application to add its own migration map.
func CommandsWithCustomMigrationMap(txConfig client.TxConfig, moduleBasics module.BasicManager, migrationMap genutiltypes.MigrationMap) *cobra.Command {
func CommandsWithCustomMigrationMap(txConfig client.TxConfig, moduleBasics module.BasicManager, appExport servertypes.AppExporter, migrationMap genutiltypes.MigrationMap) *cobra.Command {
cmd := &cobra.Command{
Use: "genesis",
Short: "Application's genesis-related subcommands",
@ -34,6 +35,7 @@ func CommandsWithCustomMigrationMap(txConfig client.TxConfig, moduleBasics modul
CollectGenTxsCmd(banktypes.GenesisBalancesIterator{}, gentxModule.GenTxValidator, txConfig.SigningContext().ValidatorAddressCodec()),
ValidateGenesisCmd(moduleBasics),
AddGenesisAccountCmd(txConfig.SigningContext().AddressCodec()),
ExportCmd(appExport),
)
return cmd

44
server/export.go → x/genutil/client/cli/export.go
View File

@ -1,4 +1,4 @@
package server
package cli
import (
"bytes"
@ -10,33 +10,34 @@ import (
"github.com/spf13/cobra"
"github.com/cosmos/cosmos-sdk/client/flags"
"github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/server"
servertypes "github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/version"
genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
)
const (
FlagHeight = "height"
FlagForZeroHeight = "for-zero-height"
FlagJailAllowedAddrs = "jail-allowed-addrs"
FlagModulesToExport = "modules-to-export"
flagTraceStore = "trace-store"
flagHeight = "height"
flagForZeroHeight = "for-zero-height"
flagJailAllowedAddrs = "jail-allowed-addrs"
flagModulesToExport = "modules-to-export"
)
// ExportCmd dumps app state to JSON.
func ExportCmd(appExporter types.AppExporter) *cobra.Command {
func ExportCmd(appExporter servertypes.AppExporter) *cobra.Command {
cmd := &cobra.Command{
Use: "export",
Short: "Export state to JSON",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
serverCtx := GetServerContextFromCmd(cmd)
config := serverCtx.Config
serverCtx := server.GetServerContextFromCmd(cmd)
if _, err := os.Stat(config.GenesisFile()); os.IsNotExist(err) {
if _, err := os.Stat(serverCtx.Config.GenesisFile()); os.IsNotExist(err) {
return err
}
db, err := openDB(config.RootDir, GetAppDBBackend(serverCtx.Viper))
db, err := server.OpenDB(serverCtx.Config.RootDir, server.GetAppDBBackend(serverCtx.Viper))
if err != nil {
return err
}
@ -50,7 +51,7 @@ func ExportCmd(appExporter types.AppExporter) *cobra.Command {
// It is possible that the genesis file is large,
// so we don't need to read it all into memory
// before we stream it out.
f, err := os.OpenFile(config.GenesisFile(), os.O_RDONLY, 0)
f, err := os.OpenFile(serverCtx.Config.GenesisFile(), os.O_RDONLY, 0)
if err != nil {
return err
}
@ -64,15 +65,16 @@ func ExportCmd(appExporter types.AppExporter) *cobra.Command {
}
traceWriterFile, _ := cmd.Flags().GetString(flagTraceStore)
traceWriter, err := openTraceWriter(traceWriterFile)
traceWriter, cleanup, err := server.SetupTraceWriter(serverCtx.Logger, traceWriterFile)
if err != nil {
return err
}
defer cleanup()
height, _ := cmd.Flags().GetInt64(FlagHeight)
forZeroHeight, _ := cmd.Flags().GetBool(FlagForZeroHeight)
jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(FlagJailAllowedAddrs)
modulesToExport, _ := cmd.Flags().GetStringSlice(FlagModulesToExport)
height, _ := cmd.Flags().GetInt64(flagHeight)
forZeroHeight, _ := cmd.Flags().GetBool(flagForZeroHeight)
jailAllowedAddrs, _ := cmd.Flags().GetStringSlice(flagJailAllowedAddrs)
modulesToExport, _ := cmd.Flags().GetStringSlice(flagModulesToExport)
outputDocument, _ := cmd.Flags().GetString(flags.FlagOutputDocument)
exported, err := appExporter(serverCtx.Logger, db, traceWriter, height, forZeroHeight, jailAllowedAddrs, serverCtx.Viper, modulesToExport)
@ -112,10 +114,10 @@ func ExportCmd(appExporter types.AppExporter) *cobra.Command {
},
}
cmd.Flags().Int64(FlagHeight, -1, "Export state from a particular height (-1 means latest height)")
cmd.Flags().Bool(FlagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)")
cmd.Flags().StringSlice(FlagJailAllowedAddrs, []string{}, "Comma-separated list of operator addresses of jailed validators to unjail")
cmd.Flags().StringSlice(FlagModulesToExport, []string{}, "Comma-separated list of modules to export. If empty, will export all modules")
cmd.Flags().Int64(flagHeight, -1, "Export state from a particular height (-1 means latest height)")
cmd.Flags().Bool(flagForZeroHeight, false, "Export state to start at height zero (perform preproccessing)")
cmd.Flags().StringSlice(flagJailAllowedAddrs, []string{}, "Comma-separated list of operator addresses of jailed validators to unjail")
cmd.Flags().StringSlice(flagModulesToExport, []string{}, "Comma-separated list of modules to export. If empty, will export all modules")
cmd.Flags().String(flags.FlagOutputDocument, "", "Exported state is written to the given file instead of STDOUT")
return cmd

8
server/export_test.go → x/genutil/client/cli/export_test.go
View File

@ -1,4 +1,4 @@
package server_test
package cli_test
import (
"context"
@ -25,7 +25,7 @@ import (
"github.com/cosmos/cosmos-sdk/server/types"
"github.com/cosmos/cosmos-sdk/testutil/cmdtest"
"github.com/cosmos/cosmos-sdk/types/module"
genutilcli "github.com/cosmos/cosmos-sdk/x/genutil/client/cli"
"github.com/cosmos/cosmos-sdk/x/genutil/client/cli"
genutiltypes "github.com/cosmos/cosmos-sdk/x/genutil/types"
)
@ -58,8 +58,8 @@ func NewExportSystem(t *testing.T, exporter types.AppExporter) *ExportSystem {
sys := cmdtest.NewSystem()
sys.AddCommands(
server.ExportCmd(exporter),
genutilcli.InitCmd(module.NewBasicManager()),
cli.ExportCmd(exporter),
cli.InitCmd(module.NewBasicManager()),
)
tw := zerolog.NewTestWriter(t)

4
x/genutil/client/cli/init_test.go
View File

@ -179,7 +179,7 @@ func TestEmptyState(t *testing.T) {
r, w, _ := os.Pipe()
os.Stdout = w
cmd = server.ExportCmd(nil)
cmd = genutilcli.ExportCmd(nil)
require.NoError(t, cmd.ExecuteContext(ctx))
outC := make(chan string)
@ -273,7 +273,7 @@ func TestInitConfig(t *testing.T) {
r, w, _ := os.Pipe()
os.Stdout = w
cmd = server.ExportCmd(nil)
cmd = genutilcli.ExportCmd(nil)
require.NoError(t, cmd.ExecuteContext(ctx))
outC := make(chan string)