tests: make integration test more stable (#1488)

* wait new blk right before send tx

* larger timeout_commit for priority test

* larger timeout_commit for mempool related test

* mv chain id test to cluster used test

* keep cluster in module scope

* sync gomod2nix

* adjust timeout_commit

* rm prune all in indexer config

* add missing min_gas_multiplier

* wait 1 more blk in upgrade

* only keep 2 validators

* add retry for grpc_eth_call

* wait 1 block before stop

* fix lint

* disable recheck

* bump up upgrade

* sync gomod2nix

* Apply suggestions from code review

* Apply suggestions from code review

* append node log

* fix lint

* expect less gas after ecd76396eb

* allow retry continue on empty rsp

* update gomod2nix

* fix flake

* mod tidy

* keep grpc only test

* tests(integration): enable recheck tx mode

* update gomod2nix

Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>
Co-authored-by: Freddy Caceres <facs95@gmail.com>
Co-authored-by: Tom <tomasguerraalda@hotmail.com>

tests/integration_tests/configs/cosmovisor.jsonnet

@@ -2,13 +2,14 @@ local config = import 'default.jsonnet';
 
 config {
   'ethermint_9000-1'+: {
+    'app-config'+: {
+      'minimum-gas-prices': '100000000000aphoton',
+    },
     genesis+: {
       app_state+: {
         feemarket+: {
           params+: {
-            no_base_fee: false,
             base_fee:: super.base_fee,
-            initial_base_fee: super.base_fee,
           },
         },
       },

tests/integration_tests/configs/default.jsonnet

@@ -4,10 +4,6 @@
     cmd: 'ethermintd',
     'start-flags': '--trace',
     config: {
-      consensus: {
-        // larger timeout for more stable mempool tests
-        timeout_commit: '2s',
-      },
       mempool: {
         // use v1 mempool to enable tx prioritization
         version: 'v1',

tests/integration_tests/configs/enable-indexer.jsonnet

@@ -8,13 +8,18 @@ config {
       },
     },
     'app-config'+: {
-      pruning: 'everything',
-      'state-sync'+: {
-        'snapshot-interval': 0,
-      },
       'json-rpc'+: {
         'enable-indexer': true,
       },
     },
+    genesis+: {
+      app_state+: {
+        feemarket+: {
+          params+: {
+            min_gas_multiplier: '0',
+          },
+        },
+      },
+    },
   },
 }

tests/integration_tests/configs/long_timeout_commit.jsonnet

@@ -0,0 +1,11 @@
+local default = import 'default.jsonnet';
+
+default {
+  'ethermint_9000-1'+: {
+    config+: {
+      consensus+: {
+        timeout_commit: '5s',
+      },
+    },
+  },
+}

tests/integration_tests/configs/pruned_node.jsonnet

@@ -8,5 +8,14 @@ config {
         'snapshot-interval': 0,
       },
     },
+    genesis+: {
+      app_state+: {
+        feemarket+: {
+          params+: {
+            min_gas_multiplier: '0',
+          },
+        },
+      },
+    },
   },
 }

tests/integration_tests/configs/rollback-test.jsonnet

@@ -2,7 +2,7 @@ local config = import 'default.jsonnet';
 
 config {
   'ethermint_9000-1'+: {
-    validators: super.validators + [{
+    validators: super.validators[0:1] + [{
       name: 'fullnode',
     }],
   },

tests/integration_tests/configs/upgrade-test-package.nix

@@ -4,9 +4,9 @@ let
   released = pkgs.buildGo118Module rec {
     name = "ethermintd";
     # the commit before https://github.com/evmos/ethermint/pull/943
-    src = fetchEthermint "f21592ebfe74da7590eb42ed926dae970b2a9a3f";
+    src = fetchEthermint "8866ae0ffd67a104e9d1cf4e50fba8391dda6c45";
     subPackages = [ "cmd/ethermintd" ];
-    vendorSha256 = "sha256-ABm5t6R/u2S6pThGrgdsqe8n3fH5tIWw7a57kxJPbYw=";
+    vendorSha256 = "sha256-oDtMamNlwe/393fZd+RNtRy6ipWpusbco8Xg1ZuKWYw=";
     doCheck = false;
   };
   current = pkgs.callPackage ../../../. { };

tests/integration_tests/conftest.py

@@ -1,8 +1,6 @@
-from pathlib import Path
-
 import pytest
 
-from .network import setup_custom_ethermint, setup_ethermint, setup_geth
+from .network import setup_ethermint, setup_geth
 
 
 @pytest.fixture(scope="session")
@@ -11,42 +9,12 @@ def ethermint(tmp_path_factory):
     yield from setup_ethermint(path, 26650)
 
 
-@pytest.fixture(scope="session")
-def ethermint_indexer(tmp_path_factory):
-    path = tmp_path_factory.mktemp("indexer")
-    yield from setup_custom_ethermint(
-        path, 26660, Path(__file__).parent / "configs/enable-indexer.jsonnet"
-    )
-
-
 @pytest.fixture(scope="session")
 def geth(tmp_path_factory):
     path = tmp_path_factory.mktemp("geth")
     yield from setup_geth(path, 8545)
 
 
-@pytest.fixture(
-    scope="session", params=["ethermint", "geth", "ethermint-ws", "enable-indexer"]
-)
-def cluster(request, ethermint, ethermint_indexer, geth):
-    """
-    run on both ethermint and geth
-    """
-    provider = request.param
-    if provider == "ethermint":
-        yield ethermint
-    elif provider == "geth":
-        yield geth
-    elif provider == "ethermint-ws":
-        ethermint_ws = ethermint.copy()
-        ethermint_ws.use_websocket()
-        yield ethermint_ws
-    elif provider == "enable-indexer":
-        yield ethermint_indexer
-    else:
-        raise NotImplementedError
-
-
 @pytest.fixture(
     scope="session", params=["ethermint", "ethermint-ws"]
 )

tests/integration_tests/cosmoscli.py

@@ -6,6 +6,7 @@ from dateutil.parser import isoparse
 from pystarport.utils import build_cli_args_safe, interact
 
 DEFAULT_GAS_PRICE = "5000000000000aphoton"
+DEFAULT_GAS = "250000"
 
 
 class ChainCommand:
@@ -636,16 +637,19 @@ class CosmosCLI:
         )
 
     def gov_propose(self, proposer, kind, proposal, **kwargs):
+        method = "submit-legacy-proposal"
         kwargs.setdefault("gas_prices", DEFAULT_GAS_PRICE)
+        kwargs.setdefault("gas", DEFAULT_GAS)
         if kind == "software-upgrade":
             return json.loads(
                 self.raw(
                     "tx",
                     "gov",
-                    "submit-proposal",
+                    method,
                     kind,
                     proposal["name"],
                     "-y",
+                    "--no-validate",
                     from_=proposer,
                     # content
                     title=proposal.get("title"),
@@ -664,7 +668,7 @@ class CosmosCLI:
                 self.raw(
                     "tx",
                     "gov",
-                    "submit-proposal",
+                    method,
                     kind,
                     "-y",
                     from_=proposer,
@@ -685,7 +689,7 @@ class CosmosCLI:
                     self.raw(
                         "tx",
                         "gov",
-                        "submit-proposal",
+                        method,
                         kind,
                         fp.name,
                         "-y",

tests/integration_tests/network.py

@@ -68,8 +68,12 @@ class Geth:
         self.w3 = w3
 
 
-def setup_ethermint(path, base_port):
-    cfg = Path(__file__).parent / "configs/default.jsonnet"
+def setup_ethermint(path, base_port, long_timeout_commit=False):
+    cfg = Path(__file__).parent / (
+        "configs/default.jsonnet"
+        if long_timeout_commit
+        else "configs/long_timeout_commit.jsonnet"
+    )
     yield from setup_custom_ethermint(path, base_port, cfg)
 
 

tests/integration_tests/test_basic.py

@@ -1,3 +0,0 @@
-def test_basic(cluster):
-    w3 = cluster.w3
-    assert w3.eth.chain_id == 9000

tests/integration_tests/test_filters.py

@@ -1,8 +1,11 @@
+from pathlib import Path
+
 import pytest
 from eth_abi import abi
 from hexbytes import HexBytes
 from web3 import Web3
 
+from .network import setup_custom_ethermint, setup_ethermint
 from .utils import (
     ADDRS,
     CONTRACTS,
@@ -12,6 +15,48 @@ from .utils import (
     w3_wait_for_new_blocks,
 )
 
+
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("filters")
+    yield from setup_ethermint(path, 26200, long_timeout_commit=True)
+
+
+@pytest.fixture(scope="module")
+def ethermint_indexer(tmp_path_factory):
+    path = tmp_path_factory.mktemp("indexer")
+    yield from setup_custom_ethermint(
+        path, 26660, Path(__file__).parent / "configs/enable-indexer.jsonnet"
+    )
+
+
+@pytest.fixture(
+    scope="module", params=["ethermint", "geth", "ethermint-ws", "enable-indexer"]
+)
+def cluster(request, custom_ethermint, ethermint_indexer, geth):
+    """
+    run on both ethermint and geth
+    """
+    provider = request.param
+    if provider == "ethermint":
+        yield custom_ethermint
+    elif provider == "geth":
+        yield geth
+    elif provider == "ethermint-ws":
+        ethermint_ws = custom_ethermint.copy()
+        ethermint_ws.use_websocket()
+        yield ethermint_ws
+    elif provider == "enable-indexer":
+        yield ethermint_indexer
+    else:
+        raise NotImplementedError
+
+
+def test_basic(cluster):
+    w3 = cluster.w3
+    assert w3.eth.chain_id == 9000
+
+
 # Smart contract names
 GREETER_CONTRACT = "Greeter"
 ERC20_CONTRACT = "TestERC20A"
@@ -58,6 +103,7 @@ def test_pending_transaction_filter(cluster):
     # without tx
     assert flt.get_new_entries() == []  # GetFilterChanges
 
+    w3_wait_for_new_blocks(w3, 1, sleep=0.1)
     # with tx
     txhash = send_successful_transaction(w3)
     assert txhash in flt.get_new_entries()

tests/integration_tests/test_grpc_only.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import subprocess
+import time
 from pathlib import Path
 
 import pytest
@@ -13,6 +14,7 @@ from .utils import (
     decode_bech32,
     deploy_contract,
     supervisorctl,
+    wait_for_block,
     wait_for_port,
 )
 
@@ -45,9 +47,6 @@ def grpc_eth_call(port: int, args: dict, chain_id=None, proposer_address=None):
     ).json()
 
 
-@pytest.mark.skip(
-    reason="undeterministic test - https://github.com/evmos/ethermint/issues/1530"
-)
 def test_grpc_mode(custom_ethermint):
     """
     - restart a fullnode in grpc-only mode
@@ -61,33 +60,43 @@ def test_grpc_mode(custom_ethermint):
         "to": contract.address,
         "data": contract.encodeABI(fn_name="currentChainID"),
     }
-    api_port = ports.api_port(custom_ethermint.base_port(2))
+    api_port = ports.api_port(custom_ethermint.base_port(1))
     # in normal mode, grpc query works even if we don't pass chain_id explicitly
-    rsp = grpc_eth_call(api_port, msg)
-    print(rsp)
-    assert "code" not in rsp, str(rsp)
-    assert 9000 == int.from_bytes(base64.b64decode(rsp["ret"].encode()), "big")
-
+    success = False
+    max_retry = 3
+    sleep = 1
+    for i in range(max_retry):
+        rsp = grpc_eth_call(api_port, msg)
+        ret = rsp["ret"]
+        valid = ret is not None
+        if valid and 9000 == int.from_bytes(base64.b64decode(ret.encode()), "big"):
+            success = True
+            break
+        time.sleep(sleep)
+    assert success
+    # wait 1 more block for both nodes to avoid node stopped before tnx get included
+    for i in range(2):
+        wait_for_block(custom_ethermint.cosmos_cli(i), 1)
     supervisorctl(
-        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node1"
     )
 
     # run grpc-only mode directly with existing chain state
-    with (custom_ethermint.base_dir / "node2.log").open("w") as logfile:
+    with (custom_ethermint.base_dir / "node1.log").open("a") as logfile:
         proc = subprocess.Popen(
             [
                 "ethermintd",
                 "start",
                 "--grpc-only",
                 "--home",
-                custom_ethermint.base_dir / "node2",
+                custom_ethermint.base_dir / "node1",
             ],
             stdout=logfile,
             stderr=subprocess.STDOUT,
         )
         try:
             # wait for grpc and rest api ports
-            grpc_port = ports.grpc_port(custom_ethermint.base_port(2))
+            grpc_port = ports.grpc_port(custom_ethermint.base_port(1))
             wait_for_port(grpc_port)
             wait_for_port(api_port)
 
@@ -102,9 +111,8 @@ def test_grpc_mode(custom_ethermint):
             assert "validator does not exist" in rsp["message"]
 
             # pass the first validator's consensus address to grpc query
-            cons_addr = decode_bech32(
-                custom_ethermint.cosmos_cli(0).consensus_address()
-            )
+            addr = custom_ethermint.cosmos_cli(0).consensus_address()
+            cons_addr = decode_bech32(addr)
 
             # should work with both chain_id and proposer_address set
             rsp = grpc_eth_call(

tests/integration_tests/test_priority.py

@@ -1,11 +1,19 @@
 import sys
 
-from .network import Ethermint
+import pytest
+
+from .network import setup_ethermint
 from .utils import ADDRS, KEYS, eth_to_bech32, sign_transaction, wait_for_new_blocks
 
 PRIORITY_REDUCTION = 1000000
 
 
+@pytest.fixture(scope="module")
+def custom_ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("priority")
+    yield from setup_ethermint(path, 26800, long_timeout_commit=True)
+
+
 def effective_gas_price(tx, base_fee):
     if "maxFeePerGas" in tx:
         # dynamic fee tx
@@ -27,7 +35,7 @@ def tx_priority(tx, base_fee):
         return (tx["gasPrice"] - base_fee) // PRIORITY_REDUCTION
 
 
-def test_priority(ethermint: Ethermint):
+def test_priority(ethermint):
     """
     test priorities of different tx types
 
@@ -112,7 +120,7 @@ def test_priority(ethermint: Ethermint):
     assert all(i1 > i2 for i1, i2 in zip(tx_indexes, tx_indexes[1:]))
 
 
-def test_native_tx_priority(ethermint: Ethermint):
+def test_native_tx_priority(ethermint):
     cli = ethermint.cosmos_cli()
     base_fee = cli.query_base_fee()
     print("base_fee", base_fee)

tests/integration_tests/test_pruned_node.py

@@ -103,7 +103,7 @@ def test_pruned_node(pruned):
     exp_tx = AttributeDict(
         {
             "from": "0x57f96e6B86CdeFdB3d412547816a82E3E0EbF9D2",
-            "gas": 51542,
+            "gas": 51406,
             "input": (
                 "0xa9059cbb000000000000000000000000378c50d9264c63f3f92b806d4ee56e"
                 "9d86ffb3ec000000000000000000000000000000000000000000000000000000"

tests/integration_tests/test_rollback.py

@@ -10,7 +10,7 @@ from .network import setup_custom_ethermint
 from .utils import supervisorctl, wait_for_block, wait_for_port
 
 
-def update_node2_cmd(path, cmd, i):
+def update_node_cmd(path, cmd, i):
     ini_path = path / SUPERVISOR_CONFIG_FILE
     ini = configparser.RawConfigParser()
     ini.read(ini_path)
@@ -29,7 +29,7 @@ def update_node2_cmd(path, cmd, i):
 def post_init(broken_binary):
     def inner(path, base_port, config):
         chain_id = "ethermint_9000-1"
-        update_node2_cmd(path / chain_id, broken_binary, 2)
+        update_node_cmd(path / chain_id, broken_binary, 1)
 
     return inner
 
@@ -66,33 +66,33 @@ def test_rollback(custom_ethermint):
     - use rollback command to rollback the db.
     - switch to correct binary should make the node syncing again.
     """
-    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+    target_port = ports.rpc_port(custom_ethermint.base_port(1))
+    wait_for_port(target_port)
 
-    print("wait for node2 to sync the first 10 blocks")
-    cli2 = custom_ethermint.cosmos_cli(2)
-    wait_for_block(cli2, 10)
+    print("wait for node1 to sync the first 10 blocks")
+    cli1 = custom_ethermint.cosmos_cli(1)
+    wait_for_block(cli1, 10)
 
     print("wait for a few more blocks on the healthy nodes")
-    cli = custom_ethermint.cosmos_cli(0)
-    wait_for_block(cli, 13)
+    cli0 = custom_ethermint.cosmos_cli(0)
+    wait_for_block(cli0, 13)
 
     # (app hash mismatch happens after the 10th block, detected in the 11th block)
-    print("check node2 get stuck at block 10")
-    assert cli2.block_height() == 10
+    print("check node1 get stuck at block 10")
+    assert cli1.block_height() == 10
 
-    print("stop node2")
+    print("stop node1")
     supervisorctl(
-        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node2"
+        custom_ethermint.base_dir / "../tasks.ini", "stop", "ethermint_9000-1-node1"
     )
 
-    print("do rollback on node2")
-    cli2.rollback()
+    print("do rollback on node1")
+    cli1.rollback()
 
     print("switch to normal binary")
-    update_node2_cmd(custom_ethermint.base_dir, "ethermintd", 2)
+    update_node_cmd(custom_ethermint.base_dir, "ethermintd", 1)
     supervisorctl(custom_ethermint.base_dir / "../tasks.ini", "update")
-    wait_for_port(ports.rpc_port(custom_ethermint.base_port(2)))
+    wait_for_port(target_port)
 
-    print("check node2 sync again")
-    cli2 = custom_ethermint.cosmos_cli(2)
-    wait_for_block(cli2, 15)
+    print("check node1 sync again")
+    wait_for_block(cli1, 15)

tests/integration_tests/test_upgrade.py

@@ -92,9 +92,6 @@ def test_cosmovisor_upgrade(custom_ethermint: Ethermint):
     - check that queries on legacy blocks still works after upgrade.
     """
     cli = custom_ethermint.cosmos_cli()
-    height = cli.block_height()
-    target_height = height + 10
-    print("upgrade height", target_height)
 
     w3 = custom_ethermint.w3
     contract, _ = deploy_contract(w3, CONTRACTS["TestERC20A"])
@@ -104,6 +101,9 @@ def test_cosmovisor_upgrade(custom_ethermint: Ethermint):
     old_erc20_balance = contract.caller.balanceOf(ADDRS["validator"])
     print("old values", old_height, old_balance, old_base_fee)
 
+    target_height = w3.eth.block_number + 10
+    print("upgrade height", target_height)
+
     plan_name = "integration-test-upgrade"
     rsp = cli.gov_propose(
         "community",
@@ -120,7 +120,6 @@ def test_cosmovisor_upgrade(custom_ethermint: Ethermint):
 
     # get proposal_id
     ev = parse_events(rsp["logs"])["submit_proposal"]
-    assert ev["proposal_type"] == "SoftwareUpgrade", rsp
     proposal_id = ev["proposal_id"]
 
     rsp = cli.gov_vote("validator", proposal_id, "yes")

tests/integration_tests/utils.py

@@ -74,13 +74,12 @@ def w3_wait_for_new_blocks(w3, n, sleep=0.5):
             break
 
 
-def wait_for_new_blocks(cli, n):
-    begin_height = int((cli.status())["SyncInfo"]["latest_block_height"])
-    while True:
-        time.sleep(0.5)
+def wait_for_new_blocks(cli, n, sleep=0.5):
+    cur_height = begin_height = int((cli.status())["SyncInfo"]["latest_block_height"])
+    while cur_height - begin_height < n:
+        time.sleep(sleep)
         cur_height = int((cli.status())["SyncInfo"]["latest_block_height"])
-        if cur_height - begin_height >= n:
-            break
+    return cur_height
 
 
 def wait_for_block(cli, height, timeout=240):