From ca070e21ef64e45a7b6d2bf8b7a1caf7aaff041a Mon Sep 17 00:00:00 2001 From: yihuang Date: Thu, 1 Sep 2022 22:22:25 +0800 Subject: [PATCH] test: add integration test for the rollback cmd (#1311) * test the fixed rollback cmd - check the rollback cmd works in integration tests * Apply suggestions from code review * upstream merged * add changelog Co-authored-by: Freddy Caceres --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- gomod2nix.toml | 5 +- server/util.go | 2 +- .../configs/broken-ethermintd.nix | 8 ++ .../configs/broken-ethermintd.patch | 15 +++ .../configs/rollback-test.jsonnet | 9 ++ tests/integration_tests/cosmoscli.py | 3 + tests/integration_tests/network.py | 9 +- tests/integration_tests/test_rollback.py | 98 +++++++++++++++++++ tests/integration_tests/utils.py | 8 ++ 12 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 tests/integration_tests/configs/broken-ethermintd.nix create mode 100644 tests/integration_tests/configs/broken-ethermintd.patch create mode 100644 tests/integration_tests/configs/rollback-test.jsonnet create mode 100644 tests/integration_tests/test_rollback.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6573e7e7..999e7ad9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (feemarket) [\#1165](https://github.com/evmos/ethermint/pull/1165) Add hint in specs about different gas terminology for gas in Cosmos and Ethereum. * (cli) [#1226](https://github.com/evmos/ethermint/pull/1226) Add custom app db backend flag. * (cli) [#1230](https://github.com/evmos/ethermint/pull/1230) Remove redundant positional height parameter from feemarket's query cli. +* (test) [#1311](https://github.com/evmos/ethermint/pull/1311) add integration test for the rollback cmd ### Bug Fixes diff --git a/go.mod b/go.mod index c59a2f21..24266cad 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/armon/go-metrics v0.4.0 github.com/btcsuite/btcd v0.22.1 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce - github.com/cosmos/cosmos-sdk v0.46.1 + github.com/cosmos/cosmos-sdk v0.46.2-0.20220831122102-a95c62680975 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/ibc-go/v5 v5.0.0-rc0 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 58512139..ce3892aa 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,8 @@ github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44= github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU= github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= -github.com/cosmos/cosmos-sdk v0.46.1 h1:7vUZXMyrmEb4xtBYpz1TobtrcnpgiZTi+tVjc0XWB4o= -github.com/cosmos/cosmos-sdk v0.46.1/go.mod h1:2+o8Qw8qnE02V+lQVZDJFQ8tri/hsiA5GmWaPERqVa0= +github.com/cosmos/cosmos-sdk v0.46.2-0.20220831122102-a95c62680975 h1:/vWkEbUtYRPvfr1qQOaggshX8wMLVfAmE3Fj1DSvNDc= +github.com/cosmos/cosmos-sdk v0.46.2-0.20220831122102-a95c62680975/go.mod h1:sCTHeC0fzLy67Jr+XyVhCYUiDHT3IHNtiWtR4WTSlbg= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= diff --git a/gomod2nix.toml b/gomod2nix.toml index 5695b2cb..ac0e2e71 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -90,8 +90,9 @@ schema = 3 version = "v1.0.0-alpha7" hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0=" [mod."github.com/cosmos/cosmos-sdk"] - version = "v0.46.1" - hash = "sha256-AZm3KzGee03chXTi6S/m0AZoGRkx4XLS6Rh2sBAWc30=" + version = "v0.46.2-0.20220831122102-a95c62680975" + hash = "sha256-S4tzkiGbaLuCzYBN/9na6LSZCJaZLmK4DIiMitozFl8=" + replaced = "github.com/cosmos/cosmos-sdk" [mod."github.com/cosmos/go-bip39"] version = "v1.0.0" hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA=" diff --git a/server/util.go b/server/util.go index 34bffd50..1450d0b6 100644 --- a/server/util.go +++ b/server/util.go @@ -50,7 +50,7 @@ func AddCommands( tendermintCmd, sdkserver.ExportCmd(appExport, defaultNodeHome), version.NewVersionCommand(), - sdkserver.NewRollbackCmd(defaultNodeHome), + sdkserver.NewRollbackCmd(appCreator, defaultNodeHome), // custom tx indexer command NewIndexTxCmd(), diff --git a/tests/integration_tests/configs/broken-ethermintd.nix b/tests/integration_tests/configs/broken-ethermintd.nix new file mode 100644 index 00000000..f789646a --- /dev/null +++ b/tests/integration_tests/configs/broken-ethermintd.nix @@ -0,0 +1,8 @@ +{ pkgs ? import ../../../nix { } }: +let ethermintd = (pkgs.callPackage ../../../. { }); +in +ethermintd.overrideAttrs (oldAttrs: { + patches = oldAttrs.patches or [ ] ++ [ + ./broken-ethermintd.patch + ]; +}) diff --git a/tests/integration_tests/configs/broken-ethermintd.patch b/tests/integration_tests/configs/broken-ethermintd.patch new file mode 100644 index 00000000..255e9949 --- /dev/null +++ b/tests/integration_tests/configs/broken-ethermintd.patch @@ -0,0 +1,15 @@ +diff --git a/app/app.go b/app/app.go +index 158bf7a3..a3b5718c 100644 +--- a/app/app.go ++++ b/app/app.go +@@ -681,6 +681,10 @@ func (app *EthermintApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBloc + + // EndBlocker updates every end block + func (app *EthermintApp) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { ++ if ctx.BlockHeight()%10 == 0 { ++ store := ctx.KVStore(app.keys["evm"]) ++ store.Set([]byte("hello"), []byte("world")) ++ } + return app.mm.EndBlock(ctx, req) + } + diff --git a/tests/integration_tests/configs/rollback-test.jsonnet b/tests/integration_tests/configs/rollback-test.jsonnet new file mode 100644 index 00000000..e39437ec --- /dev/null +++ b/tests/integration_tests/configs/rollback-test.jsonnet @@ -0,0 +1,9 @@ +local config = import 'default.jsonnet'; + +config { + 'ethermint_9000-1'+: { + validators: super.validators + [{ + name: 'fullnode', + }], + }, +} diff --git a/tests/integration_tests/cosmoscli.py b/tests/integration_tests/cosmoscli.py index de258881..54b31d11 100644 --- a/tests/integration_tests/cosmoscli.py +++ b/tests/integration_tests/cosmoscli.py @@ -826,3 +826,6 @@ class CosmosCLI: ) )["base_fee"] ) + + def rollback(self): + self.raw("rollback", home=self.data_dir) diff --git a/tests/integration_tests/network.py b/tests/integration_tests/network.py index 5a413db2..bbec87c3 100644 --- a/tests/integration_tests/network.py +++ b/tests/integration_tests/network.py @@ -96,7 +96,9 @@ def setup_geth(path, base_port): proc.wait() -def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary=None): +def setup_custom_ethermint( + path, base_port, config, post_init=None, chain_binary=None, wait_port=True +): cmd = [ "pystarport", "init", @@ -119,8 +121,9 @@ def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary preexec_fn=os.setsid, ) try: - wait_for_port(ports.evmrpc_port(base_port)) - wait_for_port(ports.evmrpc_ws_port(base_port)) + if wait_port: + wait_for_port(ports.evmrpc_port(base_port)) + wait_for_port(ports.evmrpc_ws_port(base_port)) yield Ethermint(path / "ethermint_9000-1") finally: os.killpg(os.getpgid(proc.pid), signal.SIGTERM) diff --git a/tests/integration_tests/test_rollback.py b/tests/integration_tests/test_rollback.py new file mode 100644 index 00000000..1fbe3163 --- /dev/null +++ b/tests/integration_tests/test_rollback.py @@ -0,0 +1,98 @@ +import configparser +import subprocess +from pathlib import Path + +import pytest +from pystarport import ports +from pystarport.cluster import SUPERVISOR_CONFIG_FILE + +from .network import setup_custom_ethermint +from .utils import supervisorctl, wait_for_block, wait_for_port + + +def update_node2_cmd(path, cmd, i): + ini_path = path / SUPERVISOR_CONFIG_FILE + ini = configparser.RawConfigParser() + ini.read(ini_path) + for section in ini.sections(): + if section == f"program:ethermint_9000-1-node{i}": + ini[section].update( + { + "command": f"{cmd} start --home %(here)s/node{i}", + "autorestart": "false", # don't restart when stopped + } + ) + with ini_path.open("w") as fp: + ini.write(fp) + + +def post_init(broken_binary): + def inner(path, base_port, config): + chain_id = "ethermint_9000-1" + update_node2_cmd(path / chain_id, broken_binary, 2) + + return inner + + +@pytest.fixture(scope="module") +def custom_ethermint(tmp_path_factory): + path = tmp_path_factory.mktemp("rollback") + + cmd = [ + "nix-build", + "--no-out-link", + Path(__file__).parent / "configs/broken-ethermintd.nix", + ] + print(*cmd) + broken_binary = ( + Path(subprocess.check_output(cmd).strip().decode()) / "bin/ethermintd" + ) + print(broken_binary) + + # init with genesis binary + yield from setup_custom_ethermint( + path, + 26300, + Path(__file__).parent / "configs/rollback-test.jsonnet", + post_init=post_init(broken_binary), + wait_port=False, + ) + + +def test_rollback(custom_ethermint): + """ + test using rollback command to fix app-hash mismatch situation. + - the broken node will sync up to block 10 then crash. + - use rollback command to rollback the db. + - switch to correct binary should make the node syncing again. + """ + wait_for_port(ports.rpc_port(custom_ethermint.base_port(2))) + + print("wait for node2 to sync the first 10 blocks") + cli2 = custom_ethermint.cosmos_cli(2) + wait_for_block(cli2, 10) + + print("wait for a few more blocks on the healthy nodes") + cli = custom_ethermint.cosmos_cli(0) + wait_for_block(cli, 13) + + # (app hash mismatch happens after the 10th block, detected in the 11th block) + print("check node2 get stuck at block 10") + assert cli2.block_height() == 10 + + print("stop node2") + supervisorctl( + custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2" + ) + + print("do rollback on node2") + cli2.rollback() + + print("switch to normal binary") + update_node2_cmd(custom_ethermint.base_dir, "ethermintd", 2) + supervisorctl(custom_ethermint.base_dir / "../tasks.ini", "update") + wait_for_port(ports.rpc_port(custom_ethermint.base_port(2))) + + print("check node2 sync again") + cli2 = custom_ethermint.cosmos_cli(2) + wait_for_block(cli2, 15) diff --git a/tests/integration_tests/utils.py b/tests/integration_tests/utils.py index 8b87840d..8d4af343 100644 --- a/tests/integration_tests/utils.py +++ b/tests/integration_tests/utils.py @@ -1,6 +1,7 @@ import json import os import socket +import subprocess import sys import time from pathlib import Path @@ -152,3 +153,10 @@ def send_successful_transaction(w3): def eth_to_bech32(addr, prefix=ETHERMINT_ADDRESS_PREFIX): bz = bech32.convertbits(HexBytes(addr), 8, 5) return bech32.bech32_encode(prefix, bz) + + +def supervisorctl(inipath, *args): + subprocess.run( + (sys.executable, "-msupervisor.supervisorctl", "-c", inipath, *args), + check=True, + )