test: add integration test for the rollback cmd (#1311)

* test the fixed rollback cmd

- check the rollback cmd works in integration tests

* Apply suggestions from code review

* upstream merged

* add changelog

Co-authored-by: Freddy Caceres <facs95@gmail.com>

CHANGELOG.md

@@ -59,6 +59,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (feemarket) [\#1165](https://github.com/evmos/ethermint/pull/1165) Add hint in specs about different gas terminology for gas in Cosmos and Ethereum.
 * (cli) [#1226](https://github.com/evmos/ethermint/pull/1226) Add custom app db backend flag.
 * (cli) [#1230](https://github.com/evmos/ethermint/pull/1230) Remove redundant positional height parameter from feemarket's query cli.
+* (test) [#1311](https://github.com/evmos/ethermint/pull/1311) add integration test for the rollback cmd
 
 ### Bug Fixes
 

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/armon/go-metrics v0.4.0
 	github.com/btcsuite/btcd v0.22.1
 	github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
-	github.com/cosmos/cosmos-sdk v0.46.1
+	github.com/cosmos/cosmos-sdk v0.46.2-0.20220831122102-a95c62680975
 	github.com/cosmos/go-bip39 v1.0.0
 	github.com/cosmos/ibc-go/v5 v5.0.0-rc0
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -244,8 +244,8 @@ github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44=
 github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU=
 github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0=
 github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw=
-github.com/cosmos/cosmos-sdk v0.46.1 h1:7vUZXMyrmEb4xtBYpz1TobtrcnpgiZTi+tVjc0XWB4o=
+github.com/cosmos/cosmos-sdk v0.46.2-0.20220831122102-a95c62680975 h1:/vWkEbUtYRPvfr1qQOaggshX8wMLVfAmE3Fj1DSvNDc=
-github.com/cosmos/cosmos-sdk v0.46.1/go.mod h1:2+o8Qw8qnE02V+lQVZDJFQ8tri/hsiA5GmWaPERqVa0=
+github.com/cosmos/cosmos-sdk v0.46.2-0.20220831122102-a95c62680975/go.mod h1:sCTHeC0fzLy67Jr+XyVhCYUiDHT3IHNtiWtR4WTSlbg=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
 github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=

gomod2nix.toml

@@ -90,8 +90,9 @@ schema = 3
     version = "v1.0.0-alpha7"
     hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.1"
+    version = "v0.46.2-0.20220831122102-a95c62680975"
-    hash = "sha256-AZm3KzGee03chXTi6S/m0AZoGRkx4XLS6Rh2sBAWc30="
+    hash = "sha256-S4tzkiGbaLuCzYBN/9na6LSZCJaZLmK4DIiMitozFl8="
+    replaced = "github.com/cosmos/cosmos-sdk"
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
     hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="

server/util.go

@@ -50,7 +50,7 @@ func AddCommands(
 		tendermintCmd,
 		sdkserver.ExportCmd(appExport, defaultNodeHome),
 		version.NewVersionCommand(),
-		sdkserver.NewRollbackCmd(defaultNodeHome),
+		sdkserver.NewRollbackCmd(appCreator, defaultNodeHome),
 
 		// custom tx indexer command
 		NewIndexTxCmd(),

tests/integration_tests/configs/broken-ethermintd.nix

@@ -0,0 +1,8 @@
+{ pkgs ? import ../../../nix { } }:
+let ethermintd = (pkgs.callPackage ../../../. { });
+in
+ethermintd.overrideAttrs (oldAttrs: {
+  patches = oldAttrs.patches or [ ] ++ [
+    ./broken-ethermintd.patch
+  ];
+})

tests/integration_tests/configs/broken-ethermintd.patch

@@ -0,0 +1,15 @@
+diff --git a/app/app.go b/app/app.go
+index 158bf7a3..a3b5718c 100644
+--- a/app/app.go
++++ b/app/app.go
+@@ -681,6 +681,10 @@ func (app *EthermintApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBloc
+ 
+ // EndBlocker updates every end block
+ func (app *EthermintApp) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
++	if ctx.BlockHeight()%10 == 0 {
++		store := ctx.KVStore(app.keys["evm"])
++		store.Set([]byte("hello"), []byte("world"))
++	}
+ 	return app.mm.EndBlock(ctx, req)
+ }
+ 

tests/integration_tests/configs/rollback-test.jsonnet

@@ -0,0 +1,9 @@
+local config = import 'default.jsonnet';
+
+config {
+  'ethermint_9000-1'+: {
+    validators: super.validators + [{
+      name: 'fullnode',
+    }],
+  },
+}

tests/integration_tests/cosmoscli.py

@@ -826,3 +826,6 @@ class CosmosCLI:
                 )
             )["base_fee"]
         )
+
+    def rollback(self):
+        self.raw("rollback", home=self.data_dir)

tests/integration_tests/network.py

@@ -96,7 +96,9 @@ def setup_geth(path, base_port):
             proc.wait()
 
 
-def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary=None):
+def setup_custom_ethermint(
+    path, base_port, config, post_init=None, chain_binary=None, wait_port=True
+):
     cmd = [
         "pystarport",
         "init",
@@ -119,8 +121,9 @@ def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary
         preexec_fn=os.setsid,
     )
     try:
-        wait_for_port(ports.evmrpc_port(base_port))
+        if wait_port:
-        wait_for_port(ports.evmrpc_ws_port(base_port))
+            wait_for_port(ports.evmrpc_port(base_port))
+            wait_for_port(ports.evmrpc_ws_port(base_port))
         yield Ethermint(path / "ethermint_9000-1")
     finally:
         os.killpg(os.getpgid(proc.pid), signal.SIGTERM)

tests/integration_tests/test_rollback.py

@@ -0,0 +1,98 @@
+import configparser
+import subprocess
+from pathlib import Path
+
+import pytest
+from pystarport import ports
+from pystarport.cluster import SUPERVISOR_CONFIG_FILE
+
+from .network import setup_custom_ethermint
+from .utils import supervisorctl, wait_for_block, wait_for_port
+
+
+def update_node2_cmd(path, cmd, i):
+    ini_path = path / SUPERVISOR_CONFIG_FILE
+    ini = configparser.RawConfigParser()
+    ini.read(ini_path)
+    for section in ini.sections():
+        if section == f"program:ethermint_9000-1-node{i}":
+            ini[section].update(
+                {
+                    "command": f"{cmd} start --home %(here)s/node{i}",
+                    "autorestart": "false",  # don't restart when stopped
+                }
+            )
+    with ini_path.open("w") as fp:
+        ini.write(fp)
+
+
+def post_init(broken_binary):
+    def inner(path, base_port, config):
+        chain_id = "ethermint_9000-1"
+        update_node2_cmd(path / chain_id, broken_binary, 2)
+
+    return inner
+
+
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("rollback")
+
+    cmd = [
+        "nix-build",
+        "--no-out-link",
+        Path(__file__).parent / "configs/broken-ethermintd.nix",
+    ]
+    print(*cmd)
+    broken_binary = (
+        Path(subprocess.check_output(cmd).strip().decode()) / "bin/ethermintd"
+    )
+    print(broken_binary)
+
+    # init with genesis binary
+    yield from setup_custom_ethermint(
+        path,
+        26300,
+        Path(__file__).parent / "configs/rollback-test.jsonnet",
+        post_init=post_init(broken_binary),
+        wait_port=False,
+    )
+
+
+def test_rollback(custom_ethermint):
+    """
+    test using rollback command to fix app-hash mismatch situation.
+    - the broken node will sync up to block 10 then crash.
+    - use rollback command to rollback the db.
+    - switch to correct binary should make the node syncing again.
+    """
+    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+
+    print("wait for node2 to sync the first 10 blocks")
+    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli2, 10)
+
+    print("wait for a few more blocks on the healthy nodes")
+    cli = custom_ethermint.cosmos_cli(0)
+    wait_for_block(cli, 13)
+
+    # (app hash mismatch happens after the 10th block, detected in the 11th block)
+    print("check node2 get stuck at block 10")
+    assert cli2.block_height() == 10
+
+    print("stop node2")
+    supervisorctl(
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+    )
+
+    print("do rollback on node2")
+    cli2.rollback()
+
+    print("switch to normal binary")
+    update_node2_cmd(custom_ethermint.base_dir, "ethermintd", 2)
+    supervisorctl(custom_ethermint.base_dir / "../tasks.ini", "update")
+    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+
+    print("check node2 sync again")
+    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli2, 15)

tests/integration_tests/utils.py

@@ -1,6 +1,7 @@
 import json
 import os
 import socket
+import subprocess
 import sys
 import time
 from pathlib import Path
@@ -152,3 +153,10 @@ def send_successful_transaction(w3):
 def eth_to_bech32(addr, prefix=ETHERMINT_ADDRESS_PREFIX):
     bz = bech32.convertbits(HexBytes(addr), 8, 5)
     return bech32.bech32_encode(prefix, bz)
+
+
+def supervisorctl(inipath, *args):
+    subprocess.run(
+        (sys.executable, "-msupervisor.supervisorctl", "-c", inipath, *args),
+        check=True,
+    )