test: add integration test for the rollback cmd (#1311)

* test the fixed rollback cmd

- check the rollback cmd works in integration tests

* Apply suggestions from code review

* upstream merged

* add changelog

Co-authored-by: Freddy Caceres <facs95@gmail.com>

tests/integration_tests/configs/broken-ethermintd.nix

@@ -0,0 +1,8 @@
+{ pkgs ? import ../../../nix { } }:
+let ethermintd = (pkgs.callPackage ../../../. { });
+in
+ethermintd.overrideAttrs (oldAttrs: {
+  patches = oldAttrs.patches or [ ] ++ [
+    ./broken-ethermintd.patch
+  ];
+})

tests/integration_tests/configs/broken-ethermintd.patch

@@ -0,0 +1,15 @@
+diff --git a/app/app.go b/app/app.go
+index 158bf7a3..a3b5718c 100644
+--- a/app/app.go
++++ b/app/app.go
+@@ -681,6 +681,10 @@ func (app *EthermintApp) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBloc
+ 
+ // EndBlocker updates every end block
+ func (app *EthermintApp) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
++	if ctx.BlockHeight()%10 == 0 {
++		store := ctx.KVStore(app.keys["evm"])
++		store.Set([]byte("hello"), []byte("world"))
++	}
+ 	return app.mm.EndBlock(ctx, req)
+ }
+ 

tests/integration_tests/configs/rollback-test.jsonnet

@@ -0,0 +1,9 @@
+local config = import 'default.jsonnet';
+
+config {
+  'ethermint_9000-1'+: {
+    validators: super.validators + [{
+      name: 'fullnode',
+    }],
+  },
+}

tests/integration_tests/cosmoscli.py

@@ -826,3 +826,6 @@ class CosmosCLI:
                 )
             )["base_fee"]
         )
+
+    def rollback(self):
+        self.raw("rollback", home=self.data_dir)

tests/integration_tests/network.py

@@ -96,7 +96,9 @@ def setup_geth(path, base_port):
             proc.wait()
 
 
-def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary=None):
+def setup_custom_ethermint(
+    path, base_port, config, post_init=None, chain_binary=None, wait_port=True
+):
     cmd = [
         "pystarport",
         "init",
@@ -119,8 +121,9 @@ def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary
         preexec_fn=os.setsid,
     )
     try:
-        wait_for_port(ports.evmrpc_port(base_port))
-        wait_for_port(ports.evmrpc_ws_port(base_port))
+        if wait_port:
+            wait_for_port(ports.evmrpc_port(base_port))
+            wait_for_port(ports.evmrpc_ws_port(base_port))
         yield Ethermint(path / "ethermint_9000-1")
     finally:
         os.killpg(os.getpgid(proc.pid), signal.SIGTERM)

tests/integration_tests/test_rollback.py

@@ -0,0 +1,98 @@
+import configparser
+import subprocess
+from pathlib import Path
+
+import pytest
+from pystarport import ports
+from pystarport.cluster import SUPERVISOR_CONFIG_FILE
+
+from .network import setup_custom_ethermint
+from .utils import supervisorctl, wait_for_block, wait_for_port
+
+
+def update_node2_cmd(path, cmd, i):
+    ini_path = path / SUPERVISOR_CONFIG_FILE
+    ini = configparser.RawConfigParser()
+    ini.read(ini_path)
+    for section in ini.sections():
+        if section == f"program:ethermint_9000-1-node{i}":
+            ini[section].update(
+                {
+                    "command": f"{cmd} start --home %(here)s/node{i}",
+                    "autorestart": "false",  # don't restart when stopped
+                }
+            )
+    with ini_path.open("w") as fp:
+        ini.write(fp)
+
+
+def post_init(broken_binary):
+    def inner(path, base_port, config):
+        chain_id = "ethermint_9000-1"
+        update_node2_cmd(path / chain_id, broken_binary, 2)
+
+    return inner
+
+
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("rollback")
+
+    cmd = [
+        "nix-build",
+        "--no-out-link",
+        Path(__file__).parent / "configs/broken-ethermintd.nix",
+    ]
+    print(*cmd)
+    broken_binary = (
+        Path(subprocess.check_output(cmd).strip().decode()) / "bin/ethermintd"
+    )
+    print(broken_binary)
+
+    # init with genesis binary
+    yield from setup_custom_ethermint(
+        path,
+        26300,
+        Path(__file__).parent / "configs/rollback-test.jsonnet",
+        post_init=post_init(broken_binary),
+        wait_port=False,
+    )
+
+
+def test_rollback(custom_ethermint):
+    """
+    test using rollback command to fix app-hash mismatch situation.
+    - the broken node will sync up to block 10 then crash.
+    - use rollback command to rollback the db.
+    - switch to correct binary should make the node syncing again.
+    """
+    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+
+    print("wait for node2 to sync the first 10 blocks")
+    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli2, 10)
+
+    print("wait for a few more blocks on the healthy nodes")
+    cli = custom_ethermint.cosmos_cli(0)
+    wait_for_block(cli, 13)
+
+    # (app hash mismatch happens after the 10th block, detected in the 11th block)
+    print("check node2 get stuck at block 10")
+    assert cli2.block_height() == 10
+
+    print("stop node2")
+    supervisorctl(
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+    )
+
+    print("do rollback on node2")
+    cli2.rollback()
+
+    print("switch to normal binary")
+    update_node2_cmd(custom_ethermint.base_dir, "ethermintd", 2)
+    supervisorctl(custom_ethermint.base_dir / "../tasks.ini", "update")
+    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+
+    print("check node2 sync again")
+    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli2, 15)

tests/integration_tests/utils.py

@@ -1,6 +1,7 @@
 import json
 import os
 import socket
+import subprocess
 import sys
 import time
 from pathlib import Path
@@ -152,3 +153,10 @@ def send_successful_transaction(w3):
 def eth_to_bech32(addr, prefix=ETHERMINT_ADDRESS_PREFIX):
     bz = bech32.convertbits(HexBytes(addr), 8, 5)
     return bech32.bech32_encode(prefix, bz)
+
+
+def supervisorctl(inipath, *args):
+    subprocess.run(
+        (sys.executable, "-msupervisor.supervisorctl", "-c", inipath, *args),
+        check=True,
+    )