tests: make integration test more stable (#1488)

* wait new blk right before send tx

* larger timeout_commit for priority test

* larger timeout_commit for mempool related test

* mv chain id test to cluster used test

* keep cluster in module scope

* sync gomod2nix

* adjust timeout_commit

* rm prune all in indexer config

* add missing min_gas_multiplier

* wait 1 more blk in upgrade

* only keep 2 validators

* add retry for grpc_eth_call

* wait 1 block before stop

* fix lint

* disable recheck

* bump up upgrade

* sync gomod2nix

* Apply suggestions from code review

* Apply suggestions from code review

* append node log

* fix lint

* expect less gas after ecd76396eb

* allow retry continue on empty rsp

* update gomod2nix

* fix flake

* mod tidy

* keep grpc only test

* tests(integration): enable recheck tx mode

* update gomod2nix

Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>
Co-authored-by: Freddy Caceres <facs95@gmail.com>
Co-authored-by: Tom <tomasguerraalda@hotmail.com>

gomod2nix.toml

@@ -60,8 +60,8 @@ schema = 3
     version = "v0.22.1"
     hash = "sha256-hBU+roIELcmbW2Gz7eGZzL9qNA1bakq5wNxqCgs4TKc="
   [mod."github.com/btcsuite/btcd/btcec/v2"]
-    version = "v2.2.0"
+    version = "v2.3.2"
-    hash = "sha256-cwoIqi0SiAaGEBtuQKip9OVirKIeJw5fUoWo7x/xfOc="
+    hash = "sha256-natWs+yIAuD1UI07iZtjPilroQLfXizFn3lNOiOT83U="
   [mod."github.com/btcsuite/btcd/chaincfg/chainhash"]
     version = "v1.0.1"
     hash = "sha256-vix0j/KGNvoKjhlKgVeSLY6un2FHeIEoZWMC4z3yvZ4="
@@ -91,14 +91,14 @@ schema = 3
     hash = "sha256-mFKxFHp7RaE1L5NdtH9H9px2Cy6I5uOZVnFOgE8Nrew="
     replaced = "github.com/cosmos/cosmos-sdk/ics23/go"
   [mod."github.com/cosmos/btcutil"]
-    version = "v1.0.4"
+    version = "v1.0.5"
-    hash = "sha256-JvcBXBdjdmnaW/nyf/tw/uaOAGn1b78yxrtl2/Rs3kA="
+    hash = "sha256-t572Sr5iiHcuMKLMWa2i+LBAt192oa+G1oA371tG/eI="
   [mod."github.com/cosmos/cosmos-proto"]
     version = "v1.0.0-beta.1"
     hash = "sha256-oATkuj+fM5eBn+ywO+w/tL0AFSIEkx0J3Yz+VhVe0QA="
   [mod."github.com/cosmos/cosmos-sdk"]
-    version = "v0.46.6"
+    version = "v0.46.7"
-    hash = "sha256-H1VZxZUWXhpXiY3A9smLp09MEGpXmh+XvX6YUiXPcpQ="
+    hash = "sha256-54DCF8lrnA1oUmBJlbUlWXOP5UbenRInUROn5P5I9qI="
   [mod."github.com/cosmos/go-bip39"]
     version = "v1.0.0"
     hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="
@@ -112,14 +112,11 @@ schema = 3
     version = "v0.19.4"
     hash = "sha256-EmpRZ48pjPFq/fIHneut9Vyo5QJATfb3ZO7KzWnqs9g="
   [mod."github.com/cosmos/ibc-go/v5"]
-    version = "v5.1.0"
+    version = "v5.2.0"
-    hash = "sha256-9ZwGw6XqYkBtj+vTQ2IodNb7VcGesTFnVsxjd+G1I6Q="
+    hash = "sha256-Gfqhdz9ZKEgb7LCkHiXCwYZYUYluQ+vMew/GkssfVj8="
   [mod."github.com/cosmos/ledger-cosmos-go"]
-    version = "v0.11.1"
+    version = "v0.12.1"
-    hash = "sha256-yli+VvVtZmHo2LPvCY6lYVUfcCDn3sBLDL+a8KIlqDA="
+    hash = "sha256-9+nr+/r4MyiogddS0JcXOuriPqXP4nxln8ts+mYQRcg="
-  [mod."github.com/cosmos/ledger-go"]
-    version = "v0.9.2"
-    hash = "sha256-0YI+0A6wFBjiebBwzWOQjkgXkTakCrf7Gjg2xEaQdoM="
   [mod."github.com/creachadair/taskgroup"]
     version = "v0.3.2"
     hash = "sha256-Y261IO/d9xjV0UScqHvo31broxvnKn4IQQC9Mu6jNkE="
@@ -362,11 +359,11 @@ schema = 3
     version = "v0.0.5"
     hash = "sha256-/5i70IkH/qSW5KjGzv8aQNKh9tHoz98tqtL0K2DMFn4="
   [mod."github.com/onsi/ginkgo/v2"]
-    version = "v2.5.1"
+    version = "v2.6.1"
-    hash = "sha256-VB29+H9k7l6il63oXJvsjamSUhsw/e99iI/BeTCderA="
+    hash = "sha256-OEiWYKCGPCaqL3vzSrHquHGm+Q8URT2anpanAVK5hRo="
   [mod."github.com/onsi/gomega"]
-    version = "v1.24.1"
+    version = "v1.24.2"
-    hash = "sha256-REfxQTDRcO23GnmJfOW8/MmPJf9oE2grVvvGiC1eSbo="
+    hash = "sha256-iascSzzBT1Uv/XybezSblIwwrq78BU4a9BVB5MvK6MM="
   [mod."github.com/pelletier/go-toml"]
     version = "v1.9.5"
     hash = "sha256-RJ9K1BTId0Mled7S66iGgxHkZ5JKEIsrrNaEfM8aImc="
@@ -464,8 +461,8 @@ schema = 3
     version = "v0.16.0"
     hash = "sha256-JW4zO/0vMzf1dXLePOqaMtiLUZgNbuIseh9GV+jQlf0="
   [mod."github.com/tendermint/tendermint"]
-    version = "v0.34.23"
+    version = "v0.34.24"
-    hash = "sha256-eir0vr8lQOG+FTPo+ZUeDidAIACfyUq2ie3XRE8rIa4="
+    hash = "sha256-3HFTv4XgN535RDaJ5OwUS+fnJHgkmLTwU7CNU2ilxEQ="
   [mod."github.com/tendermint/tm-db"]
     version = "v0.6.7"
     hash = "sha256-hl/3RrBrpkk2zA6dmrNlIYKs1/GfqegSscDSkA5Pjlo="
@@ -482,8 +479,11 @@ schema = 3
     version = "v0.5.8"
     hash = "sha256-bfG3dssBUn+mSOAuKL+a/DTGGLUA+eASgLoGv/Gkqs0="
   [mod."github.com/zondax/hid"]
-    version = "v0.9.1-0.20220302062450-5552068d2266"
+    version = "v0.9.1"
-    hash = "sha256-IZea8SHuLQxlltm1avieRVI054TWZg2jEoplodvNtwk="
+    hash = "sha256-hSVmN/f/lQHFhF60o6ej78ELC0MMoqQgqIX2hHjdTXg="
+  [mod."github.com/zondax/ledger-go"]
+    version = "v0.14.0"
+    hash = "sha256-RozTPSNs4RerZ4DQMBcGmvREjoRtH1G69xjhccYjIOk="
   [mod."go.etcd.io/bbolt"]
     version = "v1.3.6"
     hash = "sha256-DenVAmyN22xUiivk6fdJp4C9ZnUJXCMDUf8E0goRRV4="
@@ -497,8 +497,8 @@ schema = 3
     version = "v0.0.0-20220722155223-a9213eeb770e"
     hash = "sha256-kNgzydWRpjm0sZl4uXEs3LX5L0xjJtJRAFf/CTlYUN4="
   [mod."golang.org/x/net"]
-    version = "v0.2.0"
+    version = "v0.4.0"
-    hash = "sha256-0MqnHDdLkkau6k7hlWD9MzIoAFROvxulyT+KzZkSXOs="
+    hash = "sha256-7IwGZh/xg4mQz88cJio2Ov5d3jGRXKj1itlAja/EAbQ="
   [mod."golang.org/x/oauth2"]
     version = "v0.0.0-20221014153046-6fdb5e3db783"
     hash = "sha256-IoygidVNqyAZmN+3macDeIefK8hhJToygpcqlwehdYQ="
@@ -506,11 +506,11 @@ schema = 3
     version = "v0.1.0"
     hash = "sha256-Hygjq9euZ0qz6TvHYQwOZEjNiTbTh1nSLRAWZ6KFGR8="
   [mod."golang.org/x/sys"]
-    version = "v0.2.0"
+    version = "v0.3.0"
-    hash = "sha256-N6yfQH7R2gfcvyWTQZbxWuSNyVy6hAxiab2WFzgAykI="
+    hash = "sha256-TIHhfYbZ99sCU1ZMikxwomXH5AEtD/lA1VMMW+UAhbU="
   [mod."golang.org/x/term"]
-    version = "v0.2.0"
+    version = "v0.3.0"
-    hash = "sha256-azcllZ0o/9TurqX9udaJ0o9yxqSoI0/bSJsvQQLYIQc="
+    hash = "sha256-NKv2o8wz8DB/2W2h/muGEIHb+S06mBXZxhG254RpQ5s="
   [mod."golang.org/x/text"]
     version = "v0.5.0"
     hash = "sha256-ztH+xQyM/clOcQl+y/UEPcfNKbc3xApMbEPDDZ9up0o="

tests/integration_tests/configs/cosmovisor.jsonnet

@@ -2,13 +2,14 @@ local config = import 'default.jsonnet';
 
 config {
   'ethermint_9000-1'+: {
+    'app-config'+: {
+      'minimum-gas-prices': '100000000000aphoton',
+    },
     genesis+: {
       app_state+: {
         feemarket+: {
           params+: {
-            no_base_fee: false,
             base_fee:: super.base_fee,
-            initial_base_fee: super.base_fee,
           },
         },
       },

tests/integration_tests/configs/default.jsonnet

@@ -4,10 +4,6 @@
     cmd: 'ethermintd',
     'start-flags': '--trace',
     config: {
-      consensus: {
-        // larger timeout for more stable mempool tests
-        timeout_commit: '2s',
-      },
       mempool: {
         // use v1 mempool to enable tx prioritization
         version: 'v1',

tests/integration_tests/configs/enable-indexer.jsonnet

@@ -8,13 +8,18 @@ config {
       },
     },
     'app-config'+: {
-      pruning: 'everything',
-      'state-sync'+: {
-        'snapshot-interval': 0,
-      },
       'json-rpc'+: {
         'enable-indexer': true,
       },
     },
+    genesis+: {
+      app_state+: {
+        feemarket+: {
+          params+: {
+            min_gas_multiplier: '0',
+          },
+        },
+      },
+    },
   },
 }

tests/integration_tests/configs/long_timeout_commit.jsonnet

@@ -0,0 +1,11 @@
+local default = import 'default.jsonnet';
+
+default {
+  'ethermint_9000-1'+: {
+    config+: {
+      consensus+: {
+        timeout_commit: '5s',
+      },
+    },
+  },
+}

tests/integration_tests/configs/pruned_node.jsonnet

@@ -8,5 +8,14 @@ config {
         'snapshot-interval': 0,
       },
     },
+    genesis+: {
+      app_state+: {
+        feemarket+: {
+          params+: {
+            min_gas_multiplier: '0',
+          },
+        },
+      },
+    },
   },
 }

tests/integration_tests/configs/rollback-test.jsonnet

@@ -2,7 +2,7 @@ local config = import 'default.jsonnet';
 
 config {
   'ethermint_9000-1'+: {
-    validators: super.validators + [{
+    validators: super.validators[0:1] + [{
       name: 'fullnode',
     }],
   },

tests/integration_tests/configs/upgrade-test-package.nix

@@ -4,9 +4,9 @@ let
   released = pkgs.buildGo118Module rec {
     name = "ethermintd";
     # the commit before https://github.com/evmos/ethermint/pull/943
-    src = fetchEthermint "f21592ebfe74da7590eb42ed926dae970b2a9a3f";
+    src = fetchEthermint "8866ae0ffd67a104e9d1cf4e50fba8391dda6c45";
     subPackages = [ "cmd/ethermintd" ];
-    vendorSha256 = "sha256-ABm5t6R/u2S6pThGrgdsqe8n3fH5tIWw7a57kxJPbYw=";
+    vendorSha256 = "sha256-oDtMamNlwe/393fZd+RNtRy6ipWpusbco8Xg1ZuKWYw=";
     doCheck = false;
   };
   current = pkgs.callPackage ../../../. { };

tests/integration_tests/conftest.py

@@ -1,8 +1,6 @@
-from pathlib import Path
-
 import pytest
 
-from .network import setup_custom_ethermint, setup_ethermint, setup_geth
+from .network import setup_ethermint, setup_geth
 
 
 @pytest.fixture(scope="session")
@@ -11,42 +9,12 @@ def ethermint(tmp_path_factory):
     yield from setup_ethermint(path, 26650)
 
 
-@pytest.fixture(scope="session")
-def ethermint_indexer(tmp_path_factory):
-    path = tmp_path_factory.mktemp("indexer")
-    yield from setup_custom_ethermint(
-        path, 26660, Path(__file__).parent / "configs/enable-indexer.jsonnet"
-    )
-
-
 @pytest.fixture(scope="session")
 def geth(tmp_path_factory):
     path = tmp_path_factory.mktemp("geth")
     yield from setup_geth(path, 8545)
 
 
-@pytest.fixture(
-    scope="session", params=["ethermint", "geth", "ethermint-ws", "enable-indexer"]
-)
-def cluster(request, ethermint, ethermint_indexer, geth):
-    """
-    run on both ethermint and geth
-    """
-    provider = request.param
-    if provider == "ethermint":
-        yield ethermint
-    elif provider == "geth":
-        yield geth
-    elif provider == "ethermint-ws":
-        ethermint_ws = ethermint.copy()
-        ethermint_ws.use_websocket()
-        yield ethermint_ws
-    elif provider == "enable-indexer":
-        yield ethermint_indexer
-    else:
-        raise NotImplementedError
-
-
 @pytest.fixture(
     scope="session", params=["ethermint", "ethermint-ws"]
 )

tests/integration_tests/cosmoscli.py

@@ -6,6 +6,7 @@ from dateutil.parser import isoparse
 from pystarport.utils import build_cli_args_safe, interact
 
 DEFAULT_GAS_PRICE = "5000000000000aphoton"
+DEFAULT_GAS = "250000"
 
 
 class ChainCommand:
@@ -636,16 +637,19 @@ class CosmosCLI:
         )
 
     def gov_propose(self, proposer, kind, proposal, **kwargs):
+        method = "submit-legacy-proposal"
         kwargs.setdefault("gas_prices", DEFAULT_GAS_PRICE)
+        kwargs.setdefault("gas", DEFAULT_GAS)
         if kind == "software-upgrade":
             return json.loads(
                 self.raw(
                     "tx",
                     "gov",
-                    "submit-proposal",
+                    method,
                     kind,
                     proposal["name"],
                     "-y",
+                    "--no-validate",
                     from_=proposer,
                     # content
                     title=proposal.get("title"),
@@ -664,7 +668,7 @@ class CosmosCLI:
                 self.raw(
                     "tx",
                     "gov",
-                    "submit-proposal",
+                    method,
                     kind,
                     "-y",
                     from_=proposer,
@@ -685,7 +689,7 @@ class CosmosCLI:
                     self.raw(
                         "tx",
                         "gov",
-                        "submit-proposal",
+                        method,
                         kind,
                         fp.name,
                         "-y",

tests/integration_tests/network.py

@@ -68,8 +68,12 @@ class Geth:
         self.w3 = w3
 
 
-def setup_ethermint(path, base_port):
+def setup_ethermint(path, base_port, long_timeout_commit=False):
-    cfg = Path(__file__).parent / "configs/default.jsonnet"
+    cfg = Path(__file__).parent / (
+        "configs/default.jsonnet"
+        if long_timeout_commit
+        else "configs/long_timeout_commit.jsonnet"
+    )
     yield from setup_custom_ethermint(path, base_port, cfg)
 
 

tests/integration_tests/test_basic.py

@@ -1,3 +0,0 @@
-def test_basic(cluster):
-    w3 = cluster.w3
-    assert w3.eth.chain_id == 9000

tests/integration_tests/test_filters.py

@@ -1,8 +1,11 @@
+from pathlib import Path
+
 import pytest
 from eth_abi import abi
 from hexbytes import HexBytes
 from web3 import Web3
 
+from .network import setup_custom_ethermint, setup_ethermint
 from .utils import (
     ADDRS,
     CONTRACTS,
@@ -12,6 +15,48 @@ from .utils import (
     w3_wait_for_new_blocks,
 )
 
+
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("filters")
+    yield from setup_ethermint(path, 26200, long_timeout_commit=True)
+
+
+@pytest.fixture(scope="module")
+def ethermint_indexer(tmp_path_factory):
+    path = tmp_path_factory.mktemp("indexer")
+    yield from setup_custom_ethermint(
+        path, 26660, Path(__file__).parent / "configs/enable-indexer.jsonnet"
+    )
+
+
+@pytest.fixture(
+    scope="module", params=["ethermint", "geth", "ethermint-ws", "enable-indexer"]
+)
+def cluster(request, custom_ethermint, ethermint_indexer, geth):
+    """
+    run on both ethermint and geth
+    """
+    provider = request.param
+    if provider == "ethermint":
+        yield custom_ethermint
+    elif provider == "geth":
+        yield geth
+    elif provider == "ethermint-ws":
+        ethermint_ws = custom_ethermint.copy()
+        ethermint_ws.use_websocket()
+        yield ethermint_ws
+    elif provider == "enable-indexer":
+        yield ethermint_indexer
+    else:
+        raise NotImplementedError
+
+
+def test_basic(cluster):
+    w3 = cluster.w3
+    assert w3.eth.chain_id == 9000
+
+
 # Smart contract names
 GREETER_CONTRACT = "Greeter"
 ERC20_CONTRACT = "TestERC20A"
@@ -58,6 +103,7 @@ def test_pending_transaction_filter(cluster):
     # without tx
     assert flt.get_new_entries() == []  # GetFilterChanges
 
+    w3_wait_for_new_blocks(w3, 1, sleep=0.1)
     # with tx
     txhash = send_successful_transaction(w3)
     assert txhash in flt.get_new_entries()

tests/integration_tests/test_grpc_only.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import subprocess
+import time
 from pathlib import Path
 
 import pytest
@@ -13,6 +14,7 @@ from .utils import (
     decode_bech32,
     deploy_contract,
     supervisorctl,
+    wait_for_block,
     wait_for_port,
 )
 
@@ -45,9 +47,6 @@ def grpc_eth_call(port: int, args: dict, chain_id=None, proposer_address=None):
     ).json()
 
 
-@pytest.mark.skip(
-    reason="undeterministic test - https://github.com/evmos/ethermint/issues/1530"
-)
 def test_grpc_mode(custom_ethermint):
     """
     - restart a fullnode in grpc-only mode
@@ -61,33 +60,43 @@ def test_grpc_mode(custom_ethermint):
         "to": contract.address,
         "data": contract.encodeABI(fn_name="currentChainID"),
     }
-    api_port = ports.api_port(custom_ethermint.base_port(2))
+    api_port = ports.api_port(custom_ethermint.base_port(1))
     # in normal mode, grpc query works even if we don't pass chain_id explicitly
+    success = False
+    max_retry = 3
+    sleep = 1
+    for i in range(max_retry):
         rsp = grpc_eth_call(api_port, msg)
-    print(rsp)
+        ret = rsp["ret"]
-    assert "code" not in rsp, str(rsp)
+        valid = ret is not None
-    assert 9000 == int.from_bytes(base64.b64decode(rsp["ret"].encode()), "big")
+        if valid and 9000 == int.from_bytes(base64.b64decode(ret.encode()), "big"):
-
+            success = True
+            break
+        time.sleep(sleep)
+    assert success
+    # wait 1 more block for both nodes to avoid node stopped before tnx get included
+    for i in range(2):
+        wait_for_block(custom_ethermint.cosmos_cli(i), 1)
     supervisorctl(
-        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node1"
     )
 
     # run grpc-only mode directly with existing chain state
-    with (custom_ethermint.base_dir / "node2.log").open("w") as logfile:
+    with (custom_ethermint.base_dir / "node1.log").open("a") as logfile:
         proc = subprocess.Popen(
             [
                 "ethermintd",
                 "start",
                 "--grpc-only",
                 "--home",
-                custom_ethermint.base_dir / "node2",
+                custom_ethermint.base_dir / "node1",
             ],
             stdout=logfile,
             stderr=subprocess.STDOUT,
         )
         try:
             # wait for grpc and rest api ports
-            grpc_port = ports.grpc_port(custom_ethermint.base_port(2))
+            grpc_port = ports.grpc_port(custom_ethermint.base_port(1))
             wait_for_port(grpc_port)
             wait_for_port(api_port)
 
@@ -102,9 +111,8 @@ def test_grpc_mode(custom_ethermint):
             assert "validator does not exist" in rsp["message"]
 
             # pass the first validator's consensus address to grpc query
-            cons_addr = decode_bech32(
+            addr = custom_ethermint.cosmos_cli(0).consensus_address()
-                custom_ethermint.cosmos_cli(0).consensus_address()
+            cons_addr = decode_bech32(addr)
-            )
 
             # should work with both chain_id and proposer_address set
             rsp = grpc_eth_call(

tests/integration_tests/test_priority.py

@@ -1,11 +1,19 @@
 import sys
 
-from .network import Ethermint
+import pytest
+
+from .network import setup_ethermint
 from .utils import ADDRS, KEYS, eth_to_bech32, sign_transaction, wait_for_new_blocks
 
 PRIORITY_REDUCTION = 1000000
 
 
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("priority")
+    yield from setup_ethermint(path, 26800, long_timeout_commit=True)
+
+
 def effective_gas_price(tx, base_fee):
     if "maxFeePerGas" in tx:
         # dynamic fee tx
@@ -27,7 +35,7 @@ def tx_priority(tx, base_fee):
         return (tx["gasPrice"] - base_fee) // PRIORITY_REDUCTION
 
 
-def test_priority(ethermint: Ethermint):
+def test_priority(ethermint):
     """
     test priorities of different tx types
 
@@ -112,7 +120,7 @@ def test_priority(ethermint: Ethermint):
     assert all(i1 > i2 for i1, i2 in zip(tx_indexes, tx_indexes[1:]))
 
 
-def test_native_tx_priority(ethermint: Ethermint):
+def test_native_tx_priority(ethermint):
     cli = ethermint.cosmos_cli()
     base_fee = cli.query_base_fee()
     print("base_fee", base_fee)

tests/integration_tests/test_pruned_node.py

@@ -103,7 +103,7 @@ def test_pruned_node(pruned):
     exp_tx = AttributeDict(
         {
             "from": "0x57f96e6B86CdeFdB3d412547816a82E3E0EbF9D2",
-            "gas": 51542,
+            "gas": 51406,
             "input": (
                 "0xa9059cbb000000000000000000000000378c50d9264c63f3f92b806d4ee56e"
                 "9d86ffb3ec000000000000000000000000000000000000000000000000000000"

tests/integration_tests/test_rollback.py

@@ -10,7 +10,7 @@ from .network import setup_custom_ethermint
 from .utils import supervisorctl, wait_for_block, wait_for_port
 
 
-def update_node2_cmd(path, cmd, i):
+def update_node_cmd(path, cmd, i):
     ini_path = path / SUPERVISOR_CONFIG_FILE
     ini = configparser.RawConfigParser()
     ini.read(ini_path)
@@ -29,7 +29,7 @@ def update_node2_cmd(path, cmd, i):
 def post_init(broken_binary):
     def inner(path, base_port, config):
         chain_id = "ethermint_9000-1"
-        update_node2_cmd(path / chain_id, broken_binary, 2)
+        update_node_cmd(path / chain_id, broken_binary, 1)
 
     return inner
 
@@ -66,33 +66,33 @@ def test_rollback(custom_ethermint):
     - use rollback command to rollback the db.
     - switch to correct binary should make the node syncing again.
     """
-    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+    target_port = ports.rpc_port(custom_ethermint.base_port(1))
+    wait_for_port(target_port)
 
-    print("wait for node2 to sync the first 10 blocks")
+    print("wait for node1 to sync the first 10 blocks")
-    cli2 = custom_ethermint.cosmos_cli(2)
+    cli1 = custom_ethermint.cosmos_cli(1)
-    wait_for_block(cli2, 10)
+    wait_for_block(cli1, 10)
 
     print("wait for a few more blocks on the healthy nodes")
-    cli = custom_ethermint.cosmos_cli(0)
+    cli0 = custom_ethermint.cosmos_cli(0)
-    wait_for_block(cli, 13)
+    wait_for_block(cli0, 13)
 
     # (app hash mismatch happens after the 10th block, detected in the 11th block)
-    print("check node2 get stuck at block 10")
+    print("check node1 get stuck at block 10")
-    assert cli2.block_height() == 10
+    assert cli1.block_height() == 10
 
-    print("stop node2")
+    print("stop node1")
     supervisorctl(
-        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node1"
     )
 
-    print("do rollback on node2")
+    print("do rollback on node1")
-    cli2.rollback()
+    cli1.rollback()
 
     print("switch to normal binary")
-    update_node2_cmd(custom_ethermint.base_dir, "ethermintd", 2)
+    update_node_cmd(custom_ethermint.base_dir, "ethermintd", 1)
     supervisorctl(custom_ethermint.base_dir / "../tasks.ini", "update")
-    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+    wait_for_port(target_port)
 
-    print("check node2 sync again")
+    print("check node1 sync again")
-    cli2 = custom_ethermint.cosmos_cli(2)
+    wait_for_block(cli1, 15)
-    wait_for_block(cli2, 15)

tests/integration_tests/test_upgrade.py

@@ -92,9 +92,6 @@ def test_cosmovisor_upgrade(custom_ethermint: Ethermint):
     - check that queries on legacy blocks still works after upgrade.
     """
     cli = custom_ethermint.cosmos_cli()
-    height = cli.block_height()
-    target_height = height + 10
-    print("upgrade height", target_height)
 
     w3 = custom_ethermint.w3
     contract, _ = deploy_contract(w3, CONTRACTS["TestERC20A"])
@@ -104,6 +101,9 @@ def test_cosmovisor_upgrade(custom_ethermint: Ethermint):
     old_erc20_balance = contract.caller.balanceOf(ADDRS["validator"])
     print("old values", old_height, old_balance, old_base_fee)
 
+    target_height = w3.eth.block_number + 10
+    print("upgrade height", target_height)
+
     plan_name = "integration-test-upgrade"
     rsp = cli.gov_propose(
         "community",
@@ -120,7 +120,6 @@ def test_cosmovisor_upgrade(custom_ethermint: Ethermint):
 
     # get proposal_id
     ev = parse_events(rsp["logs"])["submit_proposal"]
-    assert ev["proposal_type"] == "SoftwareUpgrade", rsp
     proposal_id = ev["proposal_id"]
 
     rsp = cli.gov_vote("validator", proposal_id, "yes")

tests/integration_tests/utils.py

@@ -74,13 +74,12 @@ def w3_wait_for_new_blocks(w3, n, sleep=0.5):
             break
 
 
-def wait_for_new_blocks(cli, n):
+def wait_for_new_blocks(cli, n, sleep=0.5):
-    begin_height = int((cli.status())["SyncInfo"]["latest_block_height"])
+    cur_height = begin_height = int((cli.status())["SyncInfo"]["latest_block_height"])
-    while True:
+    while cur_height - begin_height < n:
-        time.sleep(0.5)
+        time.sleep(sleep)
         cur_height = int((cli.status())["SyncInfo"]["latest_block_height"])
-        if cur_height - begin_height >= n:
+    return cur_height
-            break
 
 
 def wait_for_block(cli, height, timeout=240):