LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

feat(cosmovisor): graceful shutdown (#16963)

Browse Source
This commit is contained in:
Chill Validation 2023-07-14 04:00:37 +09:00 committed by GitHub
parent 80e3c7a77c
commit 6a0ab4fd16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 190 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/cosmovisor/CHANGELOG.md
View File

@ -41,6 +41,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
* [#16413](https://github.com/cosmos/cosmos-sdk/issues/16413) Add `cosmovisor pre-upgrade` command to manually add an upgrade to cosmovisor.
* [#16573](https://github.com/cosmos/cosmos-sdk/pull/16573) Extend `cosmovisor` configuration with new log format options
* [#16550](https://github.com/cosmos/cosmos-sdk/pull/16550) Add COSMOVISOR_CUSTOM_PREUPGRADE to cosmovisor to execute custom pre-upgrade scripts (separate from daemon pre-upgrade).
* [#16963](https://github.com/cosmos/cosmos-sdk/pull/69630) Add DAEMON_SHUTDOWN_GRACE to send interrupt and wait before sending kill
* [#15361](https://github.com/cosmos/cosmos-sdk/pull/15361) Add `cosmovisor config` command to display the configuration used by cosmovisor.
## Improvements

1
tools/cosmovisor/README.md
View File

@ -90,6 +90,7 @@ Use of `cosmovisor` without one of the action arguments is deprecated. For backw
* `DAEMON_DOWNLOAD_MUST_HAVE_CHECKSUM` (*optional*, default = `false`), if `true` cosmovisor will require that a checksum is provided in the upgrade plan for the binary to be downloaded. If `false`, cosmovisor will not require a checksum to be provided, but still check the checksum if one is provided.
* `DAEMON_RESTART_AFTER_UPGRADE` (*optional*, default = `true`), if `true`, restarts the subprocess with the same command-line arguments and flags (but with the new binary) after a successful upgrade. Otherwise (`false`), `cosmovisor` stops running after an upgrade and requires the system administrator to manually restart it. Note restart is only after the upgrade and does not auto-restart the subprocess after an error occurs.
* `DAEMON_RESTART_DELAY` (*optional*, default none), allow a node operator to define a delay between the node halt (for upgrade) and backup by the specified time. The value must be a duration (e.g. `1s`).
* `DAEMON_SHUTDOWN_GRACE` (*optional*, default none), if set, send interrupt to binary and wait the specified time to allow for cleanup/cache flush to disk before sending the kill signal. The value must be a duration (e.g. `1s`).
* `DAEMON_POLL_INTERVAL` (*optional*, default 300 milliseconds), is the interval length for polling the upgrade plan file. The value must be a duration (e.g. `1s`).
* `DAEMON_DATA_BACKUP_DIR` option to set a custom backup directory. If not set, `DAEMON_HOME` is used.
* `UNSAFE_SKIP_BACKUP` (defaults to `false`), if set to `true`, upgrades directly without performing a backup. Otherwise (`false`, default) backs up the data before trying the upgrade. The default value of false is useful and recommended in case of failures and when a backup needed to rollback. We recommend using the default backup option `UNSAFE_SKIP_BACKUP=false`.

15
tools/cosmovisor/args.go
View File

@ -25,6 +25,7 @@ const (
EnvDownloadMustHaveChecksum = "DAEMON_DOWNLOAD_MUST_HAVE_CHECKSUM"
EnvRestartUpgrade = "DAEMON_RESTART_AFTER_UPGRADE"
EnvRestartDelay = "DAEMON_RESTART_DELAY"
EnvShutdownGrace = "DAEMON_SHUTDOWN_GRACE"
EnvSkipBackup = "UNSAFE_SKIP_BACKUP"
EnvDataBackupPath = "DAEMON_DATA_BACKUP_DIR"
EnvInterval = "DAEMON_POLL_INTERVAL"
@ -51,6 +52,7 @@ type Config struct {
DownloadMustHaveChecksum bool
RestartAfterUpgrade bool
RestartDelay time.Duration
ShutdownGrace time.Duration
PollInterval time.Duration
UnsafeSkipBackup bool
DataBackupPath string
@ -207,6 +209,17 @@ func GetConfigFromEnv() (*Config, error) {
}
}
cfg.ShutdownGrace = 0 // default value but makes it explicit
shutdownGrace := os.Getenv(EnvShutdownGrace)
if shutdownGrace != "" {
val, err := parseEnvDuration(shutdownGrace)
if err != nil {
errs = append(errs, fmt.Errorf("invalid: %s: %w", EnvShutdownGrace, err))
} else {
cfg.ShutdownGrace = val
}
}
envPreupgradeMaxRetriesVal := os.Getenv(EnvPreupgradeMaxRetries)
if cfg.PreupgradeMaxRetries, err = strconv.Atoi(envPreupgradeMaxRetriesVal); err != nil && envPreupgradeMaxRetriesVal != "" {
errs = append(errs, fmt.Errorf("%s could not be parsed to int: %w", EnvPreupgradeMaxRetries, err))
@ -428,6 +441,7 @@ func (cfg Config) DetailString() string {
{EnvDownloadMustHaveChecksum, fmt.Sprintf("%t", cfg.DownloadMustHaveChecksum)},
{EnvRestartUpgrade, fmt.Sprintf("%t", cfg.RestartAfterUpgrade)},
{EnvRestartDelay, cfg.RestartDelay.String()},
{EnvShutdownGrace, cfg.ShutdownGrace.String()},
{EnvInterval, cfg.PollInterval.String()},
{EnvSkipBackup, fmt.Sprintf("%t", cfg.UnsafeSkipBackup)},
{EnvDataBackupPath, cfg.DataBackupPath},
@ -436,6 +450,7 @@ func (cfg Config) DetailString() string {
{EnvColorLogs, fmt.Sprintf("%t", cfg.ColorLogs)},
{EnvTimeFormatLogs, cfg.TimeFormatLogs},
{EnvCustomPreupgrade, cfg.CustomPreupgrade},
{EnvDisableRecase, fmt.Sprintf("%t", cfg.DisableRecase)},
}
derivedEntries := []struct{ name, value string }{

143
tools/cosmovisor/args_test.go
View File

@ -39,6 +39,7 @@ type cosmovisorEnv struct {
TimeFormatLogs string
CustomPreupgrade string
DisableRecase string
ShutdownGrace string
}
type envMap struct {
@ -55,6 +56,7 @@ func (c cosmovisorEnv) ToMap() map[string]envMap {
EnvDownloadMustHaveChecksum: {val: c.DownloadMustHaveChecksum, allowEmpty: false},
EnvRestartUpgrade: {val: c.RestartUpgrade, allowEmpty: false},
EnvRestartDelay: {val: c.RestartDelay, allowEmpty: false},
EnvShutdownGrace: {val: c.ShutdownGrace, allowEmpty: false},
EnvSkipBackup: {val: c.SkipBackup, allowEmpty: false},
EnvDataBackupPath: {val: c.DataBackupPath, allowEmpty: false},
EnvInterval: {val: c.Interval, allowEmpty: false},
@ -82,6 +84,8 @@ func (c *cosmovisorEnv) Set(envVar, envVal string) {
c.RestartUpgrade = envVal
case EnvRestartDelay:
c.RestartDelay = envVal
case EnvShutdownGrace:
c.ShutdownGrace = envVal
case EnvSkipBackup:
c.SkipBackup = envVal
case EnvDataBackupPath:
@ -101,7 +105,7 @@ func (c *cosmovisorEnv) Set(envVar, envVal string) {
case EnvDisableRecase:
c.DisableRecase = envVal
default:
panic(fmt.Errorf("Unknown environment variable [%s]. Ccannot set field to [%s]. ", envVar, envVal))
panic(fmt.Errorf("Unknown environment variable [%s]. Cannot set field to [%s]. ", envVar, envVal))
}
}
@ -461,6 +465,7 @@ func (s *argsTestSuite) TestGetConfigFromEnv() {
timeFormatLogs string,
customPreUpgrade string,
disableRecase bool,
shutdownGrace int,
) *Config {
return &Config{
Home: home,
@ -478,6 +483,7 @@ func (s *argsTestSuite) TestGetConfigFromEnv() {
TimeFormatLogs: timeFormatLogs,
CustomPreupgrade: customPreUpgrade,
DisableRecase: disableRecase,
ShutdownGrace: time.Duration(shutdownGrace),
}
}
@ -503,19 +509,20 @@ func (s *argsTestSuite) TestGetConfigFromEnv() {
TimeFormatLogs: "bad",
CustomPreupgrade: "",
DisableRecase: "bad",
ShutdownGrace: "bad",
},
expectedCfg: nil,
expectedErrCount: 12,
expectedErrCount: 13,
},
{
name: "all good",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "true"},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", true),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "true", "10s"},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", true, 10000000000),
expectedErrCount: 0,
},
{
name: "nothing set",
envVals: cosmovisorEnv{"", "", "", "", "", "", "", "", "", "", "false", "false", "", "", ""},
envVals: cosmovisorEnv{"", "", "", "", "", "", "", "", "", "", "false", "false", "", "", "", ""},
expectedCfg: nil,
expectedErrCount: 3,
},
@ -523,231 +530,237 @@ func (s *argsTestSuite) TestGetConfigFromEnv() {
// timeformat tests are done in the TestTimeFormat
{
name: "download bin bad",
envVals: cosmovisorEnv{absPath, "testname", "bad", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", ""},
envVals: cosmovisorEnv{absPath, "testname", "bad", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "download bin not set",
envVals: cosmovisorEnv{absPath, "testname", "", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "", false),
envVals: cosmovisorEnv{absPath, "testname", "", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "", false, 0),
expectedErrCount: 0,
},
{
name: "download bin true",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "download bin false",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "download ensure checksum true",
envVals: cosmovisorEnv{absPath, "testname", "true", "false", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, false, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "false", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, false, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "restart upgrade bad",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "bad", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "bad", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "restart upgrade not set",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "restart upgrade true",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "true", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "true", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "restart upgrade true",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "skip unsafe backups bad",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "bad", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "bad", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "skip unsafe backups not set",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "skip unsafe backups true",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "skip unsafe backups false",
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "poll interval bad",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "bad", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "bad", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "poll interval 0",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "0", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "0", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "poll interval not set",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "", "1", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 300, 1, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 300, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "poll interval 600",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "600", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "600", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "poll interval 1s",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "1s", "1", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 1000, 1, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "1s", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 1000, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "poll interval -3m",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "-3m", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "-3m", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "restart delay bad",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "bad", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "bad", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "restart delay 0",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "0", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "0", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "restart delay not set",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 0, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 0, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "restart delay 600",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600", "false", "", "300ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600", "false", "", "300ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "restart delay 1s",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "1s", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 1000, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "1s", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 1000, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "restart delay -3m",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "-3m", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "-3m", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "prepupgrade max retries bad",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "bad", "false", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "bad", "false", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "prepupgrade max retries 0",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "0", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "0", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "prepupgrade max retries not set",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "prepupgrade max retries 5",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "false", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 5, false, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "false", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 5, false, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "disable logs bad",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "bad", "true", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "bad", "true", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "disable logs good",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "disable logs color bad",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "true", "bad", "kitchen", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "true", "bad", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "disable logs color good",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "disable logs timestamp",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, "", "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, "", "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "enable rf3339 logs timestamp",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", false),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", false, 0),
expectedErrCount: 0,
},
{
name: "invalid logs timestamp format",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "invalid", "preupgrade.sh", ""},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "invalid", "preupgrade.sh", "", ""},
expectedCfg: nil,
expectedErrCount: 1,
},
{
name: "disable recase good",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "true"},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", true),
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "true", ""},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", true, 0),
expectedErrCount: 0,
},
{
name: "disable recase bad",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "bad"},
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "bad", ""},
expectedErrCount: 1,
},
{
name: "shutdown grace good",
envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "true", "15s"},
expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", true, 15000000000),
expectedErrCount: 0,
},
}
for _, tc := range tests {

28
tools/cosmovisor/process.go
View File

@ -118,7 +118,33 @@ func (l Launcher) WaitForUpgradeOrExit(cmd *exec.Cmd) (bool, error) {
case <-l.fw.MonitorUpdate(currentUpgrade):
// upgrade - kill the process and restart
l.logger.Info("daemon shutting down in an attempt to restart")
_ = cmd.Process.Kill()
if l.cfg.ShutdownGrace > 0 {
// Interrupt signal
l.logger.Info("sent interrupt to app, waiting for exit")
_ = cmd.Process.Signal(os.Interrupt)
// Wait app exit
psChan := make(chan *os.ProcessState)
go func() {
pstate, _ := cmd.Process.Wait()
psChan <- pstate
}()
// Timeout and kill
select {
case <-psChan:
// Normal Exit
l.logger.Info("app exited normally")
case <-time.After(l.cfg.ShutdownGrace):
l.logger.Info("DAEMON_SHUTDOWN_GRACE exceeded, killing app")
// Kill after grace period
_ = cmd.Process.Kill()
}
} else {
// Default: Immediate app kill
_ = cmd.Process.Kill()
}
case err := <-cmdDone:
l.fw.Stop()
// no error -> command exits normally (eg. short command like `gaiad version`)

45
tools/cosmovisor/process_test.go
View File

@ -149,6 +149,51 @@ func (s *processTestSuite) TestLaunchProcessWithRestartDelay() {
}
}
// TestPlanShutdownGrace will test upgrades without lower case plan names
func (s *processTestSuite) TestPlanShutdownGrace() {
// binaries from testdata/validate directory
require := s.Require()
home := copyTestData(s.T(), "dontdie")
cfg := &cosmovisor.Config{Home: home, Name: "dummyd", PollInterval: 20, UnsafeSkipBackup: true, ShutdownGrace: 2 * time.Second}
logger := log.NewTestLogger(s.T()).With(log.ModuleKey, "cosmosvisor")
// should run the genesis binary and produce expected output
stdout, stderr := newBuffer(), newBuffer()
currentBin, err := cfg.CurrentBin()
require.NoError(err)
require.Equal(cfg.GenesisBin(), currentBin)
launcher, err := cosmovisor.NewLauncher(logger, cfg)
require.NoError(err)
upgradeFile := cfg.UpgradeInfoFilePath()
args := []string{"foo", "bar", "1234", upgradeFile}
doUpgrade, err := launcher.Run(args, stdout, stderr)
require.NoError(err)
require.True(doUpgrade)
require.Equal("", stderr.String())
require.Equal(fmt.Sprintf("Genesis foo bar 1234 %s\nUPGRADE \"Chain2\" NEEDED at height: 49: {}\nWARN Need Flush\nFlushed\n", upgradeFile), stdout.String())
// ensure this is upgraded now and produces new output
currentBin, err = cfg.CurrentBin()
require.NoError(err)
require.Equal(cfg.UpgradeBin("chain2"), currentBin)
args = []string{"second", "run", "--verbose"}
stdout.Reset()
stderr.Reset()
doUpgrade, err = launcher.Run(args, stdout, stderr)
require.NoError(err)
require.False(doUpgrade)
require.Equal("", stderr.String())
require.Equal("Chain 2 is live!\nArgs: second run --verbose\nFinished successfully\n", stdout.String())
// ended without other upgrade
require.Equal(cfg.UpgradeBin("chain2"), currentBin)
}
// TestLaunchProcess will try running the script a few times and watch upgrades work properly
// and args are passed through
func (s *processTestSuite) TestLaunchProcessWithDownloads() {

17
tools/cosmovisor/testdata/dontdie/cosmovisor/genesis/bin/dummyd vendored Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
warn() {
echo "WARN Need Flush"
}
trap warn INT
echo Genesis $@
sleep 1
test -z $4 && exit 1001
echo 'UPGRADE "Chain2" NEEDED at height: 49: {}'
echo '{"name":"Chain2","height":49,"info":""}' > $4
sleep 1
echo 'Flushed'
sleep 1
echo Did not kill in time. Never should be printed!!!

6
tools/cosmovisor/testdata/dontdie/cosmovisor/upgrades/chain2/bin/dummyd vendored Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
echo Chain 2 is live!
echo Args: $@
sleep 1
echo Finished successfully

0
tools/cosmovisor/testdata/dontdie/data/.gitkeep vendored Normal file
View File