From 2ee5230b2c7cf8a58516dbdf8d824dc1dc5909a9 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 10:33:06 +0200 Subject: [PATCH] feat(confix): add migration to v2 (backport #21052) (#21103) Co-authored-by: Akhil Kumar P <36399231+akhilkumarpilli@users.noreply.github.com> Co-authored-by: akhilkumarpilli --- CHANGELOG.md | 1 + store/v2/commitment/iavl/config.go | 4 +- tools/confix/cmd/migrate.go | 29 +++++-- tools/confix/cmd/mutate.go | 7 +- tools/confix/data/v2-app.toml | 59 ++++++++++++++ tools/confix/data/v2-client.toml | 27 ++++++ tools/confix/match.go | 63 ++++++++++++++ tools/confix/migrations.go | 127 +++++++++++++++++++++++++++-- tools/confix/upgrade.go | 13 ++- 9 files changed, 306 insertions(+), 24 deletions(-) create mode 100644 tools/confix/data/v2-app.toml create mode 100644 tools/confix/data/v2-client.toml create mode 100644 tools/confix/match.go diff --git a/CHANGELOG.md b/CHANGELOG.md index be7112183a..961de76b57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i ### Features +* (tools/confix) [#21052](https://github.com/cosmos/cosmos-sdk/pull/21052) Add a migration to v2 config. * (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI * (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime. * (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`. diff --git a/store/v2/commitment/iavl/config.go b/store/v2/commitment/iavl/config.go index 7e386b3a46..027f8f2788 100644 --- a/store/v2/commitment/iavl/config.go +++ b/store/v2/commitment/iavl/config.go @@ -2,8 +2,8 @@ package iavl // Config is the configuration for the IAVL tree. type Config struct { - CacheSize int `mapstructure:"cache_size"` - SkipFastStorageUpgrade bool `mapstructure:"skip_fast_storage_upgrade"` + CacheSize int `mapstructure:"cache-size" toml:"cache-size" comment:"CacheSize set the size of the iavl tree cache."` + SkipFastStorageUpgrade bool `mapstructure:"skip-fast-storage-upgrade" toml:"skip-fast-storage-upgrade" comment:"If true, the tree will work like no fast storage and always not upgrade fast storage."` } // DefaultConfig returns the default configuration for the IAVL tree. diff --git a/tools/confix/cmd/migrate.go b/tools/confix/cmd/migrate.go index ccab1f4e21..a7122161d1 100644 --- a/tools/confix/cmd/migrate.go +++ b/tools/confix/cmd/migrate.go @@ -32,15 +32,31 @@ In case of any error in updating the file, no output is written.`, RunE: func(cmd *cobra.Command, args []string) error { var configPath string clientCtx := client.GetClientContextFromCmd(cmd) + + configType := confix.AppConfigType + isClient, _ := cmd.Flags().GetBool(confix.ClientConfigType) + + if isClient { + configType = confix.ClientConfigType + } + switch { case len(args) > 1: configPath = args[1] case clientCtx.HomeDir != "": - configPath = filepath.Join(clientCtx.HomeDir, "config", "app.toml") + suffix := "app.toml" + if isClient { + suffix = "client.toml" + } + configPath = filepath.Join(clientCtx.HomeDir, "config", suffix) default: return errors.New("must provide a path to the app.toml or client.toml") } + if strings.HasSuffix(configPath, "client.toml") && !isClient { + return errors.New("app.toml file expected, got client.toml, use --client flag to migrate client.toml") + } + targetVersion := args[0] plan, ok := confix.Migrations[targetVersion] if !ok { @@ -62,15 +78,10 @@ In case of any error in updating the file, no output is written.`, outputPath = "" } - configType := confix.AppConfigType - if ok, _ := cmd.Flags().GetBool(confix.ClientConfigType); ok { - configPath = strings.ReplaceAll(configPath, "app.toml", "client.toml") // for the case we are using the home dir of client ctx - configType = confix.ClientConfigType - } else if strings.HasSuffix(configPath, "client.toml") { - return errors.New("app.toml file expected, got client.toml, use --client flag to migrate client.toml") - } + // get transformation steps and formatDoc in which plan need to be applied + steps, formatDoc := plan(rawFile, targetVersion, configType) - if err := confix.Upgrade(ctx, plan(rawFile, targetVersion, configType), configPath, outputPath, FlagSkipValidate); err != nil { + if err := confix.Upgrade(ctx, steps, formatDoc, configPath, outputPath, FlagSkipValidate); err != nil { return fmt.Errorf("failed to migrate config: %w", err) } diff --git a/tools/confix/cmd/mutate.go b/tools/confix/cmd/mutate.go index 73ae7c3d1e..772f38e7a8 100644 --- a/tools/confix/cmd/mutate.go +++ b/tools/confix/cmd/mutate.go @@ -73,7 +73,12 @@ func SetCommand() *cobra.Command { ctx = confix.WithLogWriter(ctx, cmd.ErrOrStderr()) } - return confix.Upgrade(ctx, plan, filename, outputPath, FlagSkipValidate) + doc, err := confix.LoadConfig(filename) + if err != nil { + return err + } + + return confix.Upgrade(ctx, plan, doc, filename, outputPath, FlagSkipValidate) }, } diff --git a/tools/confix/data/v2-app.toml b/tools/confix/data/v2-app.toml new file mode 100644 index 0000000000..6a93006ecc --- /dev/null +++ b/tools/confix/data/v2-app.toml @@ -0,0 +1,59 @@ +[comet] +# min-retain-blocks defines the minimum block height offset from the current block being committed, such that all blocks past this offset are pruned from CometBFT. A value of 0 indicates that no blocks should be pruned. +min-retain-blocks = 0 +# index-events defines the set of events in the form {eventType}.{attributeKey}, which informs CometBFT what to index. If empty, all events will be indexed. +index-events = [] +# halt-height contains a non-zero block height at which a node will gracefully halt and shutdown that can be used to assist upgrades and testing. +halt-height = 0 +# halt-time contains a non-zero minimum block time (in Unix seconds) at which a node will gracefully halt and shutdown that can be used to assist upgrades and testing. +halt-time = 0 +# address defines the CometBFT RPC server address to bind to. +address = 'tcp://127.0.0.1:26658' +# transport defines the CometBFT RPC server transport protocol: socket, grpc +transport = 'socket' +# trace enables the CometBFT RPC server to output trace information about its internal operations. +trace = false +# standalone starts the application without the CometBFT node. The node should be started separately. +standalone = false + +[grpc] +# Enable defines if the gRPC server should be enabled. +enable = true +# Address defines the gRPC server address to bind to. +address = 'localhost:9090' +# MaxRecvMsgSize defines the max message size in bytes the server can receive. +# The default value is 10MB. +max-recv-msg-size = 10485760 +# MaxSendMsgSize defines the max message size in bytes the server can send. +# The default value is math.MaxInt32. +max-send-msg-size = 2147483647 + +[store] +# The type of database for application and snapshots databases. +app-db-backend = 'goleveldb' + +[store.options] +# State storage database type. Currently we support: 0 for SQLite, 1 for Pebble +ss-type = 0 +# State commitment database type. Currently we support:0 for iavl, 1 for iavl v2 +sc-type = 0 + +# Pruning options for state storage +[store.options.ss-pruning-option] +# Number of recent heights to keep on disk. +keep-recent = 2 +# Height interval at which pruned heights are removed from disk. +interval = 1 + +# Pruning options for state commitment +[store.options.sc-pruning-option] +# Number of recent heights to keep on disk. +keep-recent = 2 +# Height interval at which pruned heights are removed from disk. +interval = 1 + +[store.options.iavl-config] +# CacheSize set the size of the iavl tree cache. +cache-size = 100000 +# If true, the tree will work like no fast storage and always not upgrade fast storage. +skip-fast-storage-upgrade = true diff --git a/tools/confix/data/v2-client.toml b/tools/confix/data/v2-client.toml new file mode 100644 index 0000000000..04fa4067a2 --- /dev/null +++ b/tools/confix/data/v2-client.toml @@ -0,0 +1,27 @@ +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +############################################################################### +### Client Configuration ### +############################################################################### + +# The network chain ID +chain-id = "" +# The keyring's backend, where the keys are stored (os|file|kwallet|pass|test|memory) +keyring-backend = "test" +# Default key name, if set, defines the default key to use for signing transaction when the --from flag is not specified +keyring-default-keyname = "" +# CLI output format (text|json) +output = "text" +# : to CometBFT RPC interface for this chain +node = "tcp://localhost:26657" +# Transaction broadcasting mode (sync|async) +broadcast-mode = "sync" + +# gRPC server endpoint to which the client will connect. +# It can be overwritten by the --grpc-addr flag in each command. +grpc-address = "" + +# Allow the gRPC client to connect over insecure channels. +# It can be overwritten by the --grpc-insecure flag in each command. +grpc-insecure = false diff --git a/tools/confix/match.go b/tools/confix/match.go new file mode 100644 index 0000000000..9e05d5f021 --- /dev/null +++ b/tools/confix/match.go @@ -0,0 +1,63 @@ +package confix + +import ( + "sort" + + "github.com/creachadair/tomledit" + "github.com/creachadair/tomledit/transform" +) + +// MatchKeys diffs the keyspaces of the TOML documents in files lhs and rhs. +// Comments, order, and values are ignored for comparison purposes. +// It will return in the format of map[oldKey]newKey +func MatchKeys(lhs, rhs *tomledit.Document) map[string]string { + matches := matchDocs(map[string]string{}, allKVs(lhs.Global), allKVs(rhs.Global)) + + lsec, rsec := lhs.Sections, rhs.Sections + transform.SortSectionsByName(lsec) + transform.SortSectionsByName(rsec) + + i, j := 0, 0 + for i < len(lsec) && j < len(rsec) { + switch { + case lsec[i].Name.Before(rsec[j].Name): + i++ + case rsec[j].Name.Before(lsec[i].Name): + j++ + default: + matches = matchDocs(matches, allKVs(lsec[i]), allKVs(rsec[j])) + i++ + j++ + } + } + + return matches +} + +// matchDocs get all the keys matching in lhs and rhs. +// value of keys are ignored +func matchDocs(matchesMap map[string]string, lhs, rhs []KV) map[string]string { + sort.Slice(lhs, func(i, j int) bool { + return lhs[i].Key < lhs[j].Key + }) + sort.Slice(rhs, func(i, j int) bool { + return rhs[i].Key < rhs[j].Key + }) + + i, j := 0, 0 + for i < len(lhs) && j < len(rhs) { + switch { + case lhs[i].Key < rhs[j].Key: + i++ + case lhs[i].Key > rhs[j].Key: + j++ + default: + // key exists in both lhs and rhs + matchesMap[lhs[i].Key] = rhs[j].Key + i++ + j++ + } + } + + return matchesMap +} diff --git a/tools/confix/migrations.go b/tools/confix/migrations.go index 2eece120fe..737ed6b822 100644 --- a/tools/confix/migrations.go +++ b/tools/confix/migrations.go @@ -19,7 +19,7 @@ const ( ) // MigrationMap defines a mapping from a version to a transformation plan. -type MigrationMap map[string]func(from *tomledit.Document, to, planType string) transform.Plan +type MigrationMap map[string]func(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) var Migrations = MigrationMap{ "v0.45": NoPlan, // Confix supports only the current supported SDK version. So we do not support v0.44 -> v0.45. @@ -27,11 +27,34 @@ var Migrations = MigrationMap{ "v0.47": PlanBuilder, "v0.50": PlanBuilder, "v0.52": PlanBuilder, + "v2": V2PlanBuilder, // "v0.xx.x": PlanBuilder, // add specific migration in case of configuration changes in minor versions } +type v2KeyChangesMap map[string][]string + +// list all the keys which are need to be modified in v2 +var v2KeyChanges = v2KeyChangesMap{ + "min-retain-blocks": []string{"comet.min-retain-blocks"}, + "index-events": []string{"comet.index-events"}, + "halt-height": []string{"comet.halt-height"}, + "halt-time": []string{"comet.halt-time"}, + "app-db-backend": []string{"store.app-db-backend"}, + "pruning-keep-recent": []string{ + "store.options.ss-pruning-option.keep-recent", + "store.options.sc-pruning-option.keep-recent", + }, + "pruning-interval": []string{ + "store.options.ss-pruning-option.interval", + "store.options.sc-pruning-option.interval", + }, + "iavl-cache-size": []string{"store.options.iavl-config.cache-size"}, + "iavl-disable-fastnode": []string{"store.options.iavl-config.skip-fast-storage-upgrade"}, + // Add other key mappings as needed +} + // PlanBuilder is a function that returns a transformation plan for a given diff between two files. -func PlanBuilder(from *tomledit.Document, to, planType string) transform.Plan { +func PlanBuilder(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) { plan := transform.Plan{} deletedSections := map[string]bool{} @@ -114,11 +137,105 @@ func PlanBuilder(from *tomledit.Document, to, planType string) transform.Plan { plan = append(plan, step) } - return plan + return plan, from } // NoPlan returns a no-op plan. -func NoPlan(_ *tomledit.Document, to, planType string) transform.Plan { +func NoPlan(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) { fmt.Printf("no migration needed to %s\n", to) - return transform.Plan{} + return transform.Plan{}, from +} + +// V2PlanBuilder is a function that returns a transformation plan to convert to v2 config +func V2PlanBuilder(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) { + target, err := LoadLocalConfig(to, planType) + if err != nil { + panic(fmt.Errorf("failed to parse file: %w. This file should have been valid", err)) + } + + plan := transform.Plan{} + plan = updateMatchedKeysPlan(from, target, plan) + plan = applyKeyChangesPlan(from, plan) + + return plan, target +} + +// updateMatchedKeysPlan updates all matched keys with old key values +func updateMatchedKeysPlan(from, target *tomledit.Document, plan transform.Plan) transform.Plan { + matches := MatchKeys(from, target) + for oldKey, newKey := range matches { + oldEntry := getEntry(from, oldKey) + if oldEntry == nil { + continue + } + + // check if the key "app-db-backend" exists and if its value is empty in the existing config + // If the value is empty, update the key value with the default value + // of v2 i.e., goleveldb to prevent network failures. + if isAppDBBackend(newKey, oldEntry) { + continue // lets keep app-db-backend with v2 default value + } + + // update newKey value with old entry value + step := createUpdateStep(oldKey, newKey, oldEntry) + plan = append(plan, step) + } + return plan +} + +// applyKeyChangesPlan checks if key changes are needed with the "to" version and applies them +func applyKeyChangesPlan(from *tomledit.Document, plan transform.Plan) transform.Plan { + changes := v2KeyChanges + for oldKey, newKeys := range changes { + oldEntry := getEntry(from, oldKey) + if oldEntry == nil { + continue + } + + for _, newKey := range newKeys { + // check if the key "app-db-backend" exists and if its value is empty in the existing config + // If the value is empty, update the key value with the default value + // of v2 i.e., goleveldb to prevent network failures. + if isAppDBBackend(newKey, oldEntry) { + continue // lets keep app-db-backend with v2 default value + } + + // update newKey value with old entry value + step := createUpdateStep(oldKey, newKey, oldEntry) + plan = append(plan, step) + } + } + return plan +} + +// getEntry retrieves the first entry for the given key from the document +func getEntry(doc *tomledit.Document, key string) *parser.KeyValue { + splitKeys := strings.Split(key, ".") + entry := doc.First(splitKeys...) + if entry == nil || entry.KeyValue == nil { + return nil + } + return entry.KeyValue +} + +// isAppDBBackend checks if the key is "store.app-db-backend" and the value is empty +func isAppDBBackend(key string, entry *parser.KeyValue) bool { + return key == "store.app-db-backend" && entry.Value.String() == `""` +} + +// createUpdateStep creates a transformation step to update a key with a new key value +func createUpdateStep(oldKey, newKey string, oldEntry *parser.KeyValue) transform.Step { + return transform.Step{ + Desc: fmt.Sprintf("updating %s key with %s key", oldKey, newKey), + T: transform.Func(func(_ context.Context, doc *tomledit.Document) error { + splitNewKeys := strings.Split(newKey, ".") + newEntry := doc.First(splitNewKeys...) + if newEntry == nil || newEntry.KeyValue == nil { + return nil + } + + newEntry.KeyValue.Value = oldEntry.Value + return nil + }), + } } diff --git a/tools/confix/upgrade.go b/tools/confix/upgrade.go index d6b4ff7f71..d212bbfb79 100644 --- a/tools/confix/upgrade.go +++ b/tools/confix/upgrade.go @@ -28,16 +28,11 @@ import ( // Upgrade is a convenience wrapper for calls to LoadConfig, ApplyFixes, and // CheckValid. If the caller requires more control over the behavior of the // Upgrade, call those functions directly. -func Upgrade(ctx context.Context, plan transform.Plan, configPath, outputPath string, skipValidate bool) error { +func Upgrade(ctx context.Context, plan transform.Plan, doc *tomledit.Document, configPath, outputPath string, skipValidate bool) error { if configPath == "" { return errors.New("empty input configuration path") } - doc, err := LoadConfig(configPath) - if err != nil { - return fmt.Errorf("loading config: %w", err) - } - // transforms doc and reports whether it succeeded. if err := plan.Apply(ctx, doc); err != nil { return fmt.Errorf("updating %q: %w", configPath, err) @@ -48,14 +43,18 @@ func Upgrade(ctx context.Context, plan transform.Plan, configPath, outputPath st return fmt.Errorf("formatting config: %w", err) } + // ignore validation for serverv2 by checking any default field found in doc + isServerV2 := doc.First(strings.Split("store.options.ss-pruning-option", ".")...) != nil + // allow to skip validation - if !skipValidate { + if !skipValidate && !isServerV2 { // verify that file is valid after applying fixes if err := CheckValid(configPath, buf.Bytes()); err != nil { return fmt.Errorf("updated config is invalid: %w", err) } } + var err error if outputPath == "" { _, err = os.Stdout.Write(buf.Bytes()) } else {