imp(tests): integration tests setup (#1196)

* first pass

* latest

* working tests

* github actions

* remove unnecessary change

* remove unnecessary steps

* remove unnecessary import

* remove unnecessary change

* Update .github/workflows/test.yml

Co-authored-by: yihuang <huang@crypto.com>

* update .gitignore

* update github actions

* change evm denomination

* change evm denomination

* send tests to tests folder

* Delete result

* update go version

Co-authored-by: yihuang <huang@crypto.com>
Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>

tests/integration_tests/configs/default.jsonnet

@@ -0,0 +1,82 @@
+{
+  dotenv: '../../../scripts/.env',
+  'ethermintd_777-1': {
+    cmd: 'ethermintd',
+    'start-flags': '--trace',
+    'app-config': {
+      'minimum-gas-prices': '0aphoton',
+      'index-events': ['ethereum_tx.ethereumTxHash'],
+      'json-rpc': {
+        address: '0.0.0.0:{EVMRPC_PORT}',
+        'ws-address': '0.0.0.0:{EVMRPC_PORT_WS}',
+        api: 'eth,net,web3,debug',
+        'feehistory-cap': 100,
+        'block-range-cap': 10000,
+        'logs-cap': 10000,
+      },
+    },
+    validators: [{
+      coins: '1000000000000000000stake,10000000000000000000000aphoton',
+      staked: '1000000000000000000stake',
+      mnemonic: '${VALIDATOR1_MNEMONIC}',
+    }, {
+      coins: '1000000000000000000stake,10000000000000000000000aphoton',
+      staked: '1000000000000000000stake',
+      mnemonic: '${VALIDATOR2_MNEMONIC}',
+    }],
+    accounts: [{
+      name: 'community',
+      coins: '10000000000000000000000aphoton',
+      mnemonic: '${COMMUNITY_MNEMONIC}',
+    }, {
+      name: 'signer1',
+      coins: '20000000000000000000000aphoton',
+      mnemonic: '${SIGNER1_MNEMONIC}',
+    }, {
+      name: 'signer2',
+      coins: '30000000000000000000000aphoton',
+      mnemonic: '${SIGNER2_MNEMONIC}',
+    }],
+    genesis: {
+      consensus_params: {
+        block: {
+          max_bytes: '1048576',
+          max_gas: '81500000',
+        },
+      },
+      app_state: {
+        evm: {
+          params: {
+            evm_denom: 'aphoton',
+          },
+        },
+        gov: {
+          voting_params: {
+            voting_period: '10s',
+          },
+          deposit_params: {
+            max_deposit_period: '10s',
+            min_deposit: [
+              {
+                denom: 'aphoton',
+                amount: '1',
+              },
+            ],
+          },
+        },
+        transfer: {
+          params: {
+            receive_enabled: true,
+            send_enabled: true,
+          },
+        },
+        feemarket: {
+          params: {
+            no_base_fee: false,
+            base_fee: '100000000000',
+          },
+        },
+      },
+    },
+  },
+}

tests/integration_tests/conftest.py

@@ -0,0 +1,32 @@
+import pytest
+
+from .network import setup_ethermint, setup_geth
+
+@pytest.fixture(scope="session")
+def ethermint(tmp_path_factory):
+    path = tmp_path_factory.mktemp("ethermint")
+    yield from setup_ethermint(path, 26650)
+
+
+@pytest.fixture(scope="session")
+def geth(tmp_path_factory):
+    path = tmp_path_factory.mktemp("geth")
+    yield from setup_geth(path, 8545)
+
+
+@pytest.fixture(scope="session", params=["ethermint", "geth", "ethermint-ws"])
+def cluster(request, ethermint, geth):
+    """
+    run on both ethermint and geth
+    """
+    provider = request.param
+    if provider == "ethermint":
+        yield ethermint
+    elif provider == "geth":
+        yield geth
+    elif provider == "ethermint-ws":
+        ethermint_ws = ethermint.copy()
+        ethermint_ws.use_websocket()
+        yield ethermint_ws
+    else:
+        raise NotImplementedError

tests/integration_tests/contracts/.gitignore

@@ -0,0 +1,11 @@
+node_modules
+.env
+coverage
+coverage.json
+typechain
+typechain-types
+
+#Hardhat files
+cache
+artifacts
+

tests/integration_tests/contracts/README.md

@@ -0,0 +1,13 @@
+# Sample Hardhat Project
+
+This project demonstrates a basic Hardhat use case. It comes with a sample contract, a test for that contract, and a script that deploys that contract.
+
+Try running some of the following tasks:
+
+```shell
+npx hardhat help
+npx hardhat test
+GAS_REPORT=true npx hardhat test
+npx hardhat node
+npx hardhat run scripts/deploy.js
+```

tests/integration_tests/contracts/contracts/TestERC20.sol

@@ -0,0 +1,9 @@
+pragma solidity 0.8.10;
+import "@openzeppelin/contracts/token/ERC20/ERC20.sol";
+
+contract TestERC20A is ERC20 {
+
+constructor() public ERC20("TestERC20", "Test") {
+_mint(msg.sender, 100000000000000000000000000);
+}
+}

tests/integration_tests/contracts/hardhat.config.ts

@@ -0,0 +1,23 @@
+import { HardhatUserConfig } from "hardhat/config";
+import "hardhat-typechain";
+
+const config: HardhatUserConfig = {
+  solidity: {
+    compilers: [
+      {
+        version: "0.8.10",
+        settings: {
+          optimizer: {
+            enabled: true
+          }
+        }
+      },
+    ],
+  },
+  typechain: {
+    outDir: "typechain",
+    target: "ethers-v5",
+  },
+};
+
+export default config;

tests/integration_tests/contracts/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "contracts",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "typechain": "npx hardhat typechain"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@nomiclabs/hardhat-ethers": "^2.1.0",
+    "@nomiclabs/hardhat-waffle": "^2.0.3",
+    "@openzeppelin/contracts": "^4.7.0",
+    "@typechain/ethers-v5": "^5.0.0",
+    "hardhat": "^2.10.1",
+    "hardhat-typechain": "^0.3.5",
+    "ts-generator": "^0.1.1",
+    "typechain": "^4.0.3"
+  },
+  "devDependencies": {
+    "ts-node": "^10.9.1",
+    "typescript": "^4.7.4"
+  }
+}

tests/integration_tests/contracts/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  }
+}

tests/integration_tests/network.py

@@ -0,0 +1,123 @@
+import json
+import os
+import signal
+import subprocess
+from pathlib import Path
+
+import tomlkit
+import web3
+from pystarport import ports
+from web3.middleware import geth_poa_middleware
+
+from .utils import wait_for_port
+
+
+class Ethermint:
+    def __init__(self, base_dir):
+        self._w3 = None
+        self.base_dir = base_dir
+        self.config = json.loads((base_dir / "config.json").read_text())
+        self.enable_auto_deployment = False
+        self._use_websockets = False
+
+    def copy(self):
+        return Ethermint(self.base_dir)
+
+    @property
+    def w3_http_endpoint(self, i=0):
+        port = ports.evmrpc_port(self.base_port(i))
+        return f"http://localhost:{port}"
+
+    @property
+    def w3_ws_endpoint(self, i=0):
+        port = ports.evmrpc_ws_port(self.base_port(i))
+        return f"ws://localhost:{port}"
+
+    @property
+    def w3(self, i=0):
+        if self._w3 is None:
+            if self._use_websockets:
+                self._w3 = web3.Web3(
+                    web3.providers.WebsocketProvider(self.w3_ws_endpoint)
+                )
+            else:
+                self._w3 = web3.Web3(web3.providers.HTTPProvider(self.w3_http_endpoint))
+        return self._w3
+
+    def base_port(self, i):
+        return self.config["validators"][i]["base_port"]
+
+    def node_rpc(self, i):
+        return "tcp://127.0.0.1:%d" % ports.rpc_port(self.base_port(i))
+
+    def use_websocket(self, use=True):
+        self._w3 = None
+        self._use_websockets = use
+
+
+class Geth:
+    def __init__(self, w3):
+        self.w3 = w3
+
+
+def setup_ethermint(path, base_port):
+    cfg = Path(__file__).parent / "../../scripts/ethermint-devnet.yaml"
+    yield from setup_custom_ethermint(path, base_port, cfg)
+
+def setup_geth(path, base_port):
+    with (path / "geth.log").open("w") as logfile:
+        cmd = [
+            "start-geth",
+            path,
+            "--http.port",
+            str(base_port),
+            "--port",
+            str(base_port + 1),
+        ]
+        print(*cmd)
+        proc = subprocess.Popen(
+            cmd,
+            preexec_fn=os.setsid,
+            stdout=logfile,
+            stderr=subprocess.STDOUT,
+        )
+        try:
+            wait_for_port(base_port)
+            w3 = web3.Web3(web3.providers.HTTPProvider(f"http://127.0.0.1:{base_port}"))
+            w3.middleware_onion.inject(geth_poa_middleware, layer=0)
+            yield Geth(w3)
+        finally:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            # proc.terminate()
+            proc.wait()
+
+
+def setup_custom_ethermint(path, base_port, config, post_init=None, chain_binary=None):
+    cmd = [
+        "pystarport",
+        "init",
+        "--config",
+        config,
+        "--data",
+        path,
+        "--base_port",
+        str(base_port),
+        "--no_remove",
+    ]
+    if chain_binary is not None:
+        cmd = cmd[:1] + ["--cmd", chain_binary] + cmd[1:]
+    print(*cmd)
+    subprocess.run(cmd, check=True)
+    if post_init is not None:
+        post_init(path, base_port, config)
+    proc = subprocess.Popen(
+        ["pystarport", "start", "--data", path, "--quiet"],
+        preexec_fn=os.setsid,
+    )
+    try:
+        wait_for_port(ports.evmrpc_port(base_port))
+        wait_for_port(ports.evmrpc_ws_port(base_port))
+        yield Ethermint(path / "ethermint_9000-1")
+    finally:
+        os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        proc.wait()

tests/integration_tests/pyproject.toml

@@ -0,0 +1,33 @@
+[tool.poetry]
+name = "integration_tests"
+version = "0.1.0"
+description = ""
+authors = ["chain-dev <chain@crypto.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pytest = "^7.0.1"
+pytest-github-actions-annotate-failures = "^0.1.1"
+flake8 = "^4.0.1"
+black = "^22.3.0"
+flake8-black = "^0.3.2"
+flake8-isort = "^4.1.1"
+pep8-naming = "^0.11.1"
+protobuf = "^3.13.0"
+grpcio = "^1.33.2"
+PyYAML = "^5.3.1"
+python-dateutil = "^2.8.1"
+web3 = "^5.20.1"
+eth-bloom = "^1.0.4"
+python-dotenv = "^0.19.2"
+pystarport = { git = "https://github.com/crypto-com/pystarport.git", branch = "main" }
+websockets = "^9.1"
+toml = "^0.10.2"
+pysha3 = "^1.0.2"
+jsonnet = "^0.18.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

tests/integration_tests/shell.nix

@@ -0,0 +1,12 @@
+{ system ? builtins.currentSystem, pkgs ? import ../../nix { inherit system; } }:
+pkgs.mkShell {
+  buildInputs = [
+    pkgs.jq
+    (pkgs.callPackage ../../. { }) # ethermintd
+    pkgs.start-scripts
+    pkgs.go-ethereum
+    pkgs.cosmovisor
+    pkgs.nodejs
+    pkgs.test-env
+  ];
+}

tests/integration_tests/test_basic.py

@@ -0,0 +1,3 @@
+def test_basic(cluster):
+    w3 = cluster.w3
+    assert w3.eth.chain_id == 9000

tests/integration_tests/utils.py

@@ -0,0 +1,16 @@
+import socket
+import time
+
+def wait_for_port(port, host="127.0.0.1", timeout=40.0):
+    start_time = time.perf_counter()
+    while True:
+        try:
+            with socket.create_connection((host, port), timeout=timeout):
+                break
+        except OSError as ex:
+            time.sleep(0.1)
+            if time.perf_counter() - start_time >= timeout:
+                raise TimeoutError(
+                    "Waited too long for the port {} on host {} to start accepting "
+                    "connections.".format(port, host)
+                ) from ex