LaconicNetwork/cosmos-sdk
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

feat(confix): add migration to v2 (backport #21052) (#21103)

Browse Source 
Co-authored-by: Akhil Kumar P <36399231+akhilkumarpilli@users.noreply.github.com>
Co-authored-by: akhilkumarpilli <akhilkumar7947@gmail.com>
This commit is contained in:
mergify[bot] 2024-07-30 10:33:06 +02:00 committed by GitHub
parent 7d801cc3c9
commit 2ee5230b2c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 306 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -42,6 +42,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
### Features
* (tools/confix) [#21052](https://github.com/cosmos/cosmos-sdk/pull/21052) Add a migration to v2 config.
* (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI
* (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime.
* (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`.

4
store/v2/commitment/iavl/config.go
View File

@ -2,8 +2,8 @@ package iavl
// Config is the configuration for the IAVL tree.
type Config struct {
CacheSize int `mapstructure:"cache_size"`
SkipFastStorageUpgrade bool `mapstructure:"skip_fast_storage_upgrade"`
CacheSize int `mapstructure:"cache-size" toml:"cache-size" comment:"CacheSize set the size of the iavl tree cache."`
SkipFastStorageUpgrade bool `mapstructure:"skip-fast-storage-upgrade" toml:"skip-fast-storage-upgrade" comment:"If true, the tree will work like no fast storage and always not upgrade fast storage."`
}
// DefaultConfig returns the default configuration for the IAVL tree.

29
tools/confix/cmd/migrate.go
View File

@ -32,15 +32,31 @@ In case of any error in updating the file, no output is written.`,
RunE: func(cmd *cobra.Command, args []string) error {
var configPath string
clientCtx := client.GetClientContextFromCmd(cmd)
configType := confix.AppConfigType
isClient, _ := cmd.Flags().GetBool(confix.ClientConfigType)
if isClient {
configType = confix.ClientConfigType
}
switch {
case len(args) > 1:
configPath = args[1]
case clientCtx.HomeDir != "":
configPath = filepath.Join(clientCtx.HomeDir, "config", "app.toml")
suffix := "app.toml"
if isClient {
suffix = "client.toml"
}
configPath = filepath.Join(clientCtx.HomeDir, "config", suffix)
default:
return errors.New("must provide a path to the app.toml or client.toml")
}
if strings.HasSuffix(configPath, "client.toml") && !isClient {
return errors.New("app.toml file expected, got client.toml, use --client flag to migrate client.toml")
}
targetVersion := args[0]
plan, ok := confix.Migrations[targetVersion]
if !ok {
@ -62,15 +78,10 @@ In case of any error in updating the file, no output is written.`,
outputPath = ""
}
configType := confix.AppConfigType
if ok, _ := cmd.Flags().GetBool(confix.ClientConfigType); ok {
configPath = strings.ReplaceAll(configPath, "app.toml", "client.toml") // for the case we are using the home dir of client ctx
configType = confix.ClientConfigType
} else if strings.HasSuffix(configPath, "client.toml") {
return errors.New("app.toml file expected, got client.toml, use --client flag to migrate client.toml")
}
// get transformation steps and formatDoc in which plan need to be applied
steps, formatDoc := plan(rawFile, targetVersion, configType)
if err := confix.Upgrade(ctx, plan(rawFile, targetVersion, configType), configPath, outputPath, FlagSkipValidate); err != nil {
if err := confix.Upgrade(ctx, steps, formatDoc, configPath, outputPath, FlagSkipValidate); err != nil {
return fmt.Errorf("failed to migrate config: %w", err)
}

7
tools/confix/cmd/mutate.go
View File

@ -73,7 +73,12 @@ func SetCommand() *cobra.Command {
ctx = confix.WithLogWriter(ctx, cmd.ErrOrStderr())
}
return confix.Upgrade(ctx, plan, filename, outputPath, FlagSkipValidate)
doc, err := confix.LoadConfig(filename)
if err != nil {
return err
}
return confix.Upgrade(ctx, plan, doc, filename, outputPath, FlagSkipValidate)
},
}

59
tools/confix/data/v2-app.toml Normal file
View File

@ -0,0 +1,59 @@
[comet]
# min-retain-blocks defines the minimum block height offset from the current block being committed, such that all blocks past this offset are pruned from CometBFT. A value of 0 indicates that no blocks should be pruned.
min-retain-blocks = 0
# index-events defines the set of events in the form {eventType}.{attributeKey}, which informs CometBFT what to index. If empty, all events will be indexed.
index-events = []
# halt-height contains a non-zero block height at which a node will gracefully halt and shutdown that can be used to assist upgrades and testing.
halt-height = 0
# halt-time contains a non-zero minimum block time (in Unix seconds) at which a node will gracefully halt and shutdown that can be used to assist upgrades and testing.
halt-time = 0
# address defines the CometBFT RPC server address to bind to.
address = 'tcp://127.0.0.1:26658'
# transport defines the CometBFT RPC server transport protocol: socket, grpc
transport = 'socket'
# trace enables the CometBFT RPC server to output trace information about its internal operations.
trace = false
# standalone starts the application without the CometBFT node. The node should be started separately.
standalone = false
[grpc]
# Enable defines if the gRPC server should be enabled.
enable = true
# Address defines the gRPC server address to bind to.
address = 'localhost:9090'
# MaxRecvMsgSize defines the max message size in bytes the server can receive.
# The default value is 10MB.
max-recv-msg-size = 10485760
# MaxSendMsgSize defines the max message size in bytes the server can send.
# The default value is math.MaxInt32.
max-send-msg-size = 2147483647
[store]
# The type of database for application and snapshots databases.
app-db-backend = 'goleveldb'
[store.options]
# State storage database type. Currently we support: 0 for SQLite, 1 for Pebble
ss-type = 0
# State commitment database type. Currently we support:0 for iavl, 1 for iavl v2
sc-type = 0
# Pruning options for state storage
[store.options.ss-pruning-option]
# Number of recent heights to keep on disk.
keep-recent = 2
# Height interval at which pruned heights are removed from disk.
interval = 1
# Pruning options for state commitment
[store.options.sc-pruning-option]
# Number of recent heights to keep on disk.
keep-recent = 2
# Height interval at which pruned heights are removed from disk.
interval = 1
[store.options.iavl-config]
# CacheSize set the size of the iavl tree cache.
cache-size = 100000
# If true, the tree will work like no fast storage and always not upgrade fast storage.
skip-fast-storage-upgrade = true

27
tools/confix/data/v2-client.toml Normal file
View File

@ -0,0 +1,27 @@
# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml
###############################################################################
### Client Configuration ###
###############################################################################
# The network chain ID
chain-id = ""
# The keyring's backend, where the keys are stored (os|file|kwallet|pass|test|memory)
keyring-backend = "test"
# Default key name, if set, defines the default key to use for signing transaction when the --from flag is not specified
keyring-default-keyname = ""
# CLI output format (text|json)
output = "text"
# <host>:<port> to CometBFT RPC interface for this chain
node = "tcp://localhost:26657"
# Transaction broadcasting mode (sync|async)
broadcast-mode = "sync"
# gRPC server endpoint to which the client will connect.
# It can be overwritten by the --grpc-addr flag in each command.
grpc-address = ""
# Allow the gRPC client to connect over insecure channels.
# It can be overwritten by the --grpc-insecure flag in each command.
grpc-insecure = false

63
tools/confix/match.go Normal file
View File

@ -0,0 +1,63 @@
package confix
import (
"sort"
"github.com/creachadair/tomledit"
"github.com/creachadair/tomledit/transform"
)
// MatchKeys diffs the keyspaces of the TOML documents in files lhs and rhs.
// Comments, order, and values are ignored for comparison purposes.
// It will return in the format of map[oldKey]newKey
func MatchKeys(lhs, rhs *tomledit.Document) map[string]string {
matches := matchDocs(map[string]string{}, allKVs(lhs.Global), allKVs(rhs.Global))
lsec, rsec := lhs.Sections, rhs.Sections
transform.SortSectionsByName(lsec)
transform.SortSectionsByName(rsec)
i, j := 0, 0
for i < len(lsec) && j < len(rsec) {
switch {
case lsec[i].Name.Before(rsec[j].Name):
i++
case rsec[j].Name.Before(lsec[i].Name):
j++
default:
matches = matchDocs(matches, allKVs(lsec[i]), allKVs(rsec[j]))
i++
j++
}
}
return matches
}
// matchDocs get all the keys matching in lhs and rhs.
// value of keys are ignored
func matchDocs(matchesMap map[string]string, lhs, rhs []KV) map[string]string {
sort.Slice(lhs, func(i, j int) bool {
return lhs[i].Key < lhs[j].Key
})
sort.Slice(rhs, func(i, j int) bool {
return rhs[i].Key < rhs[j].Key
})
i, j := 0, 0
for i < len(lhs) && j < len(rhs) {
switch {
case lhs[i].Key < rhs[j].Key:
i++
case lhs[i].Key > rhs[j].Key:
j++
default:
// key exists in both lhs and rhs
matchesMap[lhs[i].Key] = rhs[j].Key
i++
j++
}
}
return matchesMap
}

127
tools/confix/migrations.go
View File

@ -19,7 +19,7 @@ const (
)
// MigrationMap defines a mapping from a version to a transformation plan.
type MigrationMap map[string]func(from *tomledit.Document, to, planType string) transform.Plan
type MigrationMap map[string]func(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document)
var Migrations = MigrationMap{
"v0.45": NoPlan, // Confix supports only the current supported SDK version. So we do not support v0.44 -> v0.45.
@ -27,11 +27,34 @@ var Migrations = MigrationMap{
"v0.47": PlanBuilder,
"v0.50": PlanBuilder,
"v0.52": PlanBuilder,
"v2": V2PlanBuilder,
// "v0.xx.x": PlanBuilder, // add specific migration in case of configuration changes in minor versions
}
type v2KeyChangesMap map[string][]string
// list all the keys which are need to be modified in v2
var v2KeyChanges = v2KeyChangesMap{
"min-retain-blocks": []string{"comet.min-retain-blocks"},
"index-events": []string{"comet.index-events"},
"halt-height": []string{"comet.halt-height"},
"halt-time": []string{"comet.halt-time"},
"app-db-backend": []string{"store.app-db-backend"},
"pruning-keep-recent": []string{
"store.options.ss-pruning-option.keep-recent",
"store.options.sc-pruning-option.keep-recent",
},
"pruning-interval": []string{
"store.options.ss-pruning-option.interval",
"store.options.sc-pruning-option.interval",
},
"iavl-cache-size": []string{"store.options.iavl-config.cache-size"},
"iavl-disable-fastnode": []string{"store.options.iavl-config.skip-fast-storage-upgrade"},
// Add other key mappings as needed
}
// PlanBuilder is a function that returns a transformation plan for a given diff between two files.
func PlanBuilder(from *tomledit.Document, to, planType string) transform.Plan {
func PlanBuilder(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) {
plan := transform.Plan{}
deletedSections := map[string]bool{}
@ -114,11 +137,105 @@ func PlanBuilder(from *tomledit.Document, to, planType string) transform.Plan {
plan = append(plan, step)
}
return plan
return plan, from
}
// NoPlan returns a no-op plan.
func NoPlan(_ *tomledit.Document, to, planType string) transform.Plan {
func NoPlan(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) {
fmt.Printf("no migration needed to %s\n", to)
return transform.Plan{}
return transform.Plan{}, from
}
// V2PlanBuilder is a function that returns a transformation plan to convert to v2 config
func V2PlanBuilder(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) {
target, err := LoadLocalConfig(to, planType)
if err != nil {
panic(fmt.Errorf("failed to parse file: %w. This file should have been valid", err))
}
plan := transform.Plan{}
plan = updateMatchedKeysPlan(from, target, plan)
plan = applyKeyChangesPlan(from, plan)
return plan, target
}
// updateMatchedKeysPlan updates all matched keys with old key values
func updateMatchedKeysPlan(from, target *tomledit.Document, plan transform.Plan) transform.Plan {
matches := MatchKeys(from, target)
for oldKey, newKey := range matches {
oldEntry := getEntry(from, oldKey)
if oldEntry == nil {
continue
}
// check if the key "app-db-backend" exists and if its value is empty in the existing config
// If the value is empty, update the key value with the default value
// of v2 i.e., goleveldb to prevent network failures.
if isAppDBBackend(newKey, oldEntry) {
continue // lets keep app-db-backend with v2 default value
}
// update newKey value with old entry value
step := createUpdateStep(oldKey, newKey, oldEntry)
plan = append(plan, step)
}
return plan
}
// applyKeyChangesPlan checks if key changes are needed with the "to" version and applies them
func applyKeyChangesPlan(from *tomledit.Document, plan transform.Plan) transform.Plan {
changes := v2KeyChanges
for oldKey, newKeys := range changes {
oldEntry := getEntry(from, oldKey)
if oldEntry == nil {
continue
}
for _, newKey := range newKeys {
// check if the key "app-db-backend" exists and if its value is empty in the existing config
// If the value is empty, update the key value with the default value
// of v2 i.e., goleveldb to prevent network failures.
if isAppDBBackend(newKey, oldEntry) {
continue // lets keep app-db-backend with v2 default value
}
// update newKey value with old entry value
step := createUpdateStep(oldKey, newKey, oldEntry)
plan = append(plan, step)
}
}
return plan
}
// getEntry retrieves the first entry for the given key from the document
func getEntry(doc *tomledit.Document, key string) *parser.KeyValue {
splitKeys := strings.Split(key, ".")
entry := doc.First(splitKeys...)
if entry == nil || entry.KeyValue == nil {
return nil
}
return entry.KeyValue
}
// isAppDBBackend checks if the key is "store.app-db-backend" and the value is empty
func isAppDBBackend(key string, entry *parser.KeyValue) bool {
return key == "store.app-db-backend" && entry.Value.String() == `""`
}
// createUpdateStep creates a transformation step to update a key with a new key value
func createUpdateStep(oldKey, newKey string, oldEntry *parser.KeyValue) transform.Step {
return transform.Step{
Desc: fmt.Sprintf("updating %s key with %s key", oldKey, newKey),
T: transform.Func(func(_ context.Context, doc *tomledit.Document) error {
splitNewKeys := strings.Split(newKey, ".")
newEntry := doc.First(splitNewKeys...)
if newEntry == nil || newEntry.KeyValue == nil {
return nil
}
newEntry.KeyValue.Value = oldEntry.Value
return nil
}),
}
}

13
tools/confix/upgrade.go
View File

@ -28,16 +28,11 @@ import (
// Upgrade is a convenience wrapper for calls to LoadConfig, ApplyFixes, and
// CheckValid. If the caller requires more control over the behavior of the
// Upgrade, call those functions directly.
func Upgrade(ctx context.Context, plan transform.Plan, configPath, outputPath string, skipValidate bool) error {
func Upgrade(ctx context.Context, plan transform.Plan, doc *tomledit.Document, configPath, outputPath string, skipValidate bool) error {
if configPath == "" {
return errors.New("empty input configuration path")
}
doc, err := LoadConfig(configPath)
if err != nil {
return fmt.Errorf("loading config: %w", err)
}
// transforms doc and reports whether it succeeded.
if err := plan.Apply(ctx, doc); err != nil {
return fmt.Errorf("updating %q: %w", configPath, err)
@ -48,14 +43,18 @@ func Upgrade(ctx context.Context, plan transform.Plan, configPath, outputPath st
return fmt.Errorf("formatting config: %w", err)
}
// ignore validation for serverv2 by checking any default field found in doc
isServerV2 := doc.First(strings.Split("store.options.ss-pruning-option", ".")...) != nil
// allow to skip validation
if !skipValidate {
if !skipValidate && !isServerV2 {
// verify that file is valid after applying fixes
if err := CheckValid(configPath, buf.Bytes()); err != nil {
return fmt.Errorf("updated config is invalid: %w", err)
}
}
var err error
if outputPath == "" {
_, err = os.Stdout.Write(buf.Bytes())
} else {