Initial prototype of prb-math external tests using foundry rewritten in python

Co-authored-by: Kamil Śliwak <kamil.sliwak@codepoets.it>

.circleci/config.yml

@@ -691,7 +691,7 @@ defaults:
       name: t_native_test_ext_prb_math
       project: prb-math
       binary_type: native
-      image: cimg/node:18.16
+      image: cimg/rust:1.70
 
   - job_native_test_ext_elementfi: &job_native_test_ext_elementfi
       <<: *requires_b_ubu_static
@@ -1724,11 +1724,13 @@ workflows:
             - t_native_test_ext_yield_liquidator
             - t_native_test_ext_perpetual_pools
             - t_native_test_ext_uniswap
-            - t_native_test_ext_prb_math
             - t_native_test_ext_elementfi
             - t_native_test_ext_brink
             # NOTE: We are disabling gp2 tests due to constant failures.
             #- t_native_test_ext_gp2
+            # TODO: Dropping prb-math from the benchmarks since it is not implemented yet
+            # in the new Foundry external testing infrastructure.
+            # - t_native_test_ext_prb_math
             # NOTE: The external tests below were commented because they
             # depend on a specific version of hardhat which does not support shanghai EVM.
             #- t_native_test_ext_trident

scripts/externalTests/runners/base.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import subprocess
+from abc import ABCMeta
+from abc import abstractmethod
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+from shutil import rmtree
+from tempfile import mkdtemp
+from textwrap import dedent
+from typing import List
+from typing import Set
+
+from test_helpers import download_project
+from test_helpers import get_solc_short_version
+from test_helpers import parse_command_line
+from test_helpers import parse_custom_presets
+from test_helpers import parse_solc_version
+from test_helpers import replace_version_pragmas
+from test_helpers import settings_from_preset
+from test_helpers import SettingsPreset
+
+CURRENT_EVM_VERSION: str = "shanghai"
+
+@dataclass
+class TestConfig:
+    name: str
+    repo_url: str
+    ref_type: str
+    ref: str
+    compile_only_presets: List[SettingsPreset] = field(default_factory=list)
+    settings_presets: List[SettingsPreset] = field(default_factory=lambda: list(SettingsPreset))
+    evm_version: str = field(default=CURRENT_EVM_VERSION)
+
+    def selected_presets(self) -> Set[SettingsPreset]:
+        return set(self.compile_only_presets + self.settings_presets)
+
+
+class BaseRunner(metaclass=ABCMeta):
+    config: TestConfig
+    solc_binary_type: str
+    solc_binary_path: Path
+    presets: Set[SettingsPreset]
+
+    def __init__(self, argv, config: TestConfig):
+        args = parse_command_line(f"{config.name} external tests", argv)
+        self.config = config
+        self.solc_binary_type = args.solc_binary_type
+        self.solc_binary_path = args.solc_binary_path
+        self.presets = parse_custom_presets(args.selected_presets) if args.selected_presets else config.selected_presets()
+        self.env = os.environ.copy()
+        self.tmp_dir = mkdtemp(prefix=f"ext-test-{config.name}-")
+        self.test_dir = Path(self.tmp_dir) / "ext"
+
+    def setup_solc(self) -> str:
+        if self.solc_binary_type == "solcjs":
+            # TODO: add support to solc-js
+            raise NotImplementedError()
+        print("Setting up solc...")
+        solc_version_output = subprocess.check_output(
+            [self.solc_binary_path, "--version"],
+            shell=False,
+            encoding="utf-8"
+        ).split(":")[1]
+        return parse_solc_version(solc_version_output)
+
+    @staticmethod
+    def enter_test_dir(fn):
+        """Run a function inside the test directory"""
+
+        previous_dir = os.getcwd()
+        def f(self, *args, **kwargs):
+            try:
+                assert self.test_dir is not None
+                os.chdir(self.test_dir)
+                return fn(self, *args, **kwargs)
+            finally:
+                # Restore the previous directory after execute fn
+                os.chdir(previous_dir)
+        return f
+
+    def setup_environment(self):
+        """Configure the project build environment"""
+        print("Configuring Runner building environment...")
+        replace_version_pragmas(self.test_dir)
+
+    @enter_test_dir
+    def clean(self):
+        """Clean temporary directories"""
+        rmtree(self.tmp_dir)
+
+    @enter_test_dir
+    @abstractmethod
+    def configure(self):
+        raise NotImplementedError()
+
+    @enter_test_dir
+    @abstractmethod
+    def compile(self, preset: SettingsPreset):
+        raise NotImplementedError()
+
+    @enter_test_dir
+    @abstractmethod
+    def run_test(self):
+        raise NotImplementedError()
+
+def run_test(runner: BaseRunner):
+    print(f"Testing {runner.config.name}...\n===========================")
+    print(f"Selected settings presets: {' '.join(p.value for p in runner.presets)}")
+
+    # Configure solc compiler
+    solc_version = runner.setup_solc()
+    print(f"Using compiler version {solc_version}")
+
+    # Download project
+    download_project(runner.test_dir, runner.config.repo_url, runner.config.ref_type, runner.config.ref)
+
+    # Configure run environment
+    runner.setup_environment()
+
+    # Configure TestRunner instance
+    print(dedent(f"""\
+        Configuring runner's profiles with:
+        -------------------------------------
+        Binary type: {runner.solc_binary_type}
+        Compiler path: {runner.solc_binary_path}
+        -------------------------------------
+    """))
+    runner.configure()
+    for preset in runner.presets:
+        print("Running compile function...")
+        settings = settings_from_preset(preset, runner.config.evm_version)
+        print(dedent(f"""\
+            -------------------------------------
+            Settings preset: {preset.value}
+            Settings: {settings}
+            EVM version: {runner.config.evm_version}
+            Compiler version: {get_solc_short_version(solc_version)}
+            Compiler version (full): {solc_version}
+            -------------------------------------
+        """))
+        runner.compile(preset)
+        # TODO: COMPILE_ONLY should be a command-line option
+        if os.environ.get("COMPILE_ONLY") == "1" or preset in runner.config.compile_only_presets:
+            print("Skipping test function...")
+        else:
+            print("Running test function...")
+            runner.run_test()
+        # TODO: store_benchmark_report
+    runner.clean()
+    print("Done.")

scripts/externalTests/runners/foundry.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import re
+import subprocess
+from shutil import which
+from textwrap import dedent
+from typing import Optional
+
+from runners.base import BaseRunner
+from test_helpers import SettingsPreset
+from test_helpers import settings_from_preset
+
+def run_forge_command(command: str, env: Optional[dict] = None):
+    subprocess.run(
+        command.split(),
+        env=env if env is not None else os.environ.copy(),
+        check=True
+    )
+
+
+class FoundryRunner(BaseRunner):
+    """Configure and run Foundry-based projects"""
+
+    FOUNDRY_CONFIG_FILE = "foundry.toml"
+
+    def setup_environment(self):
+        super().setup_environment()
+        if which("forge") is None:
+            raise RuntimeError("Forge not found.")
+
+    @staticmethod
+    def profile_name(preset: SettingsPreset):
+        """Returns foundry profile name"""
+        # Replace - or + by underscore to avoid invalid toml syntax
+        return re.sub(r"(\-|\+)+", "_", preset.value)
+
+    @staticmethod
+    def profile_section(profile_fields: dict) -> str:
+        return dedent("""\
+            [profile.{name}]
+            gas_reports = ["*"]
+            auto_detect_solc = false
+            solc = "{solc}"
+            evm_version = "{evm_version}"
+            optimizer = {optimizer}
+            via_ir = {via_ir}
+
+            [profile.{name}.optimizer_details]
+            yul = {yul}
+        """).format(**profile_fields)
+
+    @BaseRunner.enter_test_dir
+    def configure(self):
+        """Configure forge tests profiles"""
+
+        profiles = []
+        for preset in self.presets:
+            settings = settings_from_preset(preset, self.config.evm_version)
+            profiles.append(self.profile_section({
+                "name": self.profile_name(preset),
+                "solc": self.solc_binary_path,
+                "evm_version": self.config.evm_version,
+                "optimizer": str(settings["optimizer"]["enabled"]).lower(),
+                "via_ir": str(settings["viaIR"]).lower(),
+                "yul": str(settings["optimizer"]["details"]["yul"]).lower(),
+            }))
+
+        with open(
+            file=self.test_dir / self.FOUNDRY_CONFIG_FILE,
+            mode="a",
+            encoding="utf-8",
+        ) as f:
+            for profile in profiles:
+                f.write(profile)
+
+        run_forge_command("forge install", self.env)
+
+    @BaseRunner.enter_test_dir
+    def compile(self, preset: SettingsPreset):
+        """Compile project"""
+
+        # Set the Foundry profile environment variable
+        self.env.update({"FOUNDRY_PROFILE": self.profile_name(preset)})
+        run_forge_command("forge build", self.env)
+
+    @BaseRunner.enter_test_dir
+    def run_test(self):
+        """Run project tests"""
+
+        run_forge_command("forge test --gas-report", self.env)

scripts/externalTests/test_helpers.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import re
+import subprocess
+import sys
+from argparse import ArgumentParser
+from enum import Enum
+from pathlib import Path
+from typing import List
+from typing import Set
+
+# Our scripts/ is not a proper Python package so we need to modify PYTHONPATH to import from it
+# pragma pylint: disable=import-error,wrong-import-position
+PROJECT_ROOT = Path(__file__).parents[2]
+sys.path.insert(0, f"{PROJECT_ROOT}/scripts/common")
+
+from git_helpers import git_commit_hash
+
+SOLC_FULL_VERSION_REGEX = re.compile(r"^[a-zA-Z: ]*(.*)$")
+SOLC_SHORT_VERSION_REGEX = re.compile(r"^([0-9.]+).*\+|\-$")
+
+
+class SettingsPreset(Enum):
+    LEGACY_NO_OPTIMIZE = 'legacy-no-optimize'
+    IR_NO_OPTIMIZE = 'ir-no-optimize'
+    LEGACY_OPTIMIZE_EVM_ONLY = 'legacy-optimize-evm-only'
+    IR_OPTIMIZE_EVM_ONLY = 'ir-optimize-evm-only'
+    LEGACY_OPTIMIZE_EVM_YUL = 'legacy-optimize-evm+yul'
+    IR_OPTIMIZE_EVM_YUL = 'ir-optimize-evm+yul'
+
+
+def compiler_settings(evm_version: str, via_ir: bool = False, optimizer: bool = False, yul: bool = False) -> dict:
+    return {
+        "optimizer": {"enabled": optimizer, "details": {"yul": yul}},
+        "evmVersion": evm_version,
+        "viaIR": via_ir,
+    }
+
+
+def settings_from_preset(preset: SettingsPreset, evm_version: str) -> dict:
+    return {
+        SettingsPreset.LEGACY_NO_OPTIMIZE:       compiler_settings(evm_version),
+        SettingsPreset.IR_NO_OPTIMIZE:           compiler_settings(evm_version, via_ir=True),
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_ONLY: compiler_settings(evm_version, optimizer=True),
+        SettingsPreset.IR_OPTIMIZE_EVM_ONLY:     compiler_settings(evm_version, via_ir=True, optimizer=True),
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_YUL:  compiler_settings(evm_version, optimizer=True, yul=True),
+        SettingsPreset.IR_OPTIMIZE_EVM_YUL:      compiler_settings(evm_version, via_ir=True, optimizer=True, yul=True),
+    }[preset]
+
+
+def parse_custom_presets(presets: List[str]) -> Set[SettingsPreset]:
+    return {SettingsPreset(p) for p in presets}
+
+def parse_command_line(description: str, args: List[str]):
+    arg_parser = ArgumentParser(description)
+    arg_parser.add_argument(
+        "solc_binary_type",
+        metavar="solc-binary-type",
+        type=str,
+        default="native",
+        choices=["native", "solcjs"],
+        help="""Solidity compiler binary type""",
+    )
+    arg_parser.add_argument(
+        "solc_binary_path",
+        metavar="solc-binary-path",
+        type=Path,
+        help="""Path to solc binary""",
+    )
+    arg_parser.add_argument(
+        "selected_presets",
+        metavar="selected-presets",
+        help="""List of compiler settings presets""",
+        nargs='*',
+    )
+    return arg_parser.parse_args(args)
+
+
+def download_project(test_dir: Path, repo_url: str, ref_type: str = "branch", ref: str = "master"):
+    assert ref_type in ("commit", "branch", "tag")
+
+    print(f"Cloning {ref_type} {ref} of {repo_url}...")
+    if ref_type == "commit":
+        os.mkdir(test_dir)
+        os.chdir(test_dir)
+        subprocess.run(["git", "init"], check=True)
+        subprocess.run(["git", "remote", "add", "origin", repo_url], check=True)
+        subprocess.run(["git", "fetch", "--depth", "1", "origin", ref], check=True)
+        subprocess.run(["git", "reset", "--hard", "FETCH_HEAD"], check=True)
+    else:
+        os.chdir(test_dir.parent)
+        subprocess.run(["git", "clone", "--no-progress", "--depth", "1", repo_url, "-b", ref, test_dir.resolve()], check=True)
+        if not test_dir.exists():
+            raise RuntimeError("Failed to clone the project.")
+        os.chdir(test_dir)
+
+    if (test_dir / ".gitmodules").exists():
+        subprocess.run(["git", "submodule", "update", "--init"], check=True)
+
+    print(f"Current commit hash: {git_commit_hash()}")
+
+
+def parse_solc_version(solc_version_string: str) -> str:
+    solc_version_match = re.search(SOLC_FULL_VERSION_REGEX, solc_version_string)
+    if solc_version_match is None:
+        raise RuntimeError(f"Solc version could not be found in: {solc_version_string}.")
+    return solc_version_match.group(1)
+
+
+def get_solc_short_version(solc_full_version: str) -> str:
+    solc_short_version_match = re.search(SOLC_SHORT_VERSION_REGEX, solc_full_version)
+    if solc_short_version_match is None:
+        raise RuntimeError(f"Error extracting short version string from: {solc_full_version}.")
+    return solc_short_version_match.group(1)
+
+
+def store_benchmark_report(self):
+    raise NotImplementedError()
+
+
+def replace_version_pragmas(test_dir: Path):
+    """
+    Replace fixed-version pragmas (part of Consensys best practice).
+    Include all directories to also cover node dependencies.
+    """
+    print("Replacing fixed-version pragmas...")
+    for source in test_dir.glob("**/*.sol"):
+        content = source.read_text(encoding="utf-8")
+        content = re.sub(r"pragma solidity [^;]+;", r"pragma solidity >=0.0;", content)
+        with open(source, "w", encoding="utf-8") as f:
+            f.write(content)

test/externalTests/prb-math.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import sys
+from pathlib import Path
+
+# Our scripts/ is not a proper Python package so we need to modify PYTHONPATH to import from it
+# pragma pylint: disable=import-error,wrong-import-position
+PROJECT_ROOT = Path(__file__).parents[2]
+sys.path.insert(0, f"{PROJECT_ROOT}/scripts/externalTests")
+
+from runners.base import run_test
+from runners.base import TestConfig
+from runners.foundry import FoundryRunner
+from test_helpers import SettingsPreset
+
+test_config = TestConfig(
+    name="PRBMath",
+    repo_url="https://github.com/PaulRBerg/prb-math.git",
+    ref_type="branch",
+    ref="main",
+    compile_only_presets=[
+        # pylint: disable=line-too-long
+        # SettingsPreset.IR_NO_OPTIMIZE,       # Error: Yul exception:Variable expr_15699_address is 2 slot(s) too deep inside the stack. Stack too deep.
+        # SettingsPreset.IR_OPTIMIZE_EVM_ONLY, # Error: Yul exception:Variable expr_15699_address is 2 slot(s) too deep inside the stack. Stack too deep.
+    ],
+    settings_presets=[
+        SettingsPreset.LEGACY_NO_OPTIMIZE,
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_ONLY,
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_YUL,
+        SettingsPreset.IR_OPTIMIZE_EVM_YUL,
+    ],
+)
+
+sys.exit(run_test(FoundryRunner(argv=sys.argv[1:], config=test_config)))

test/external_tests.py

@@ -36,7 +36,7 @@ def detect_external_tests() -> dict:
     return {
         file_path.stem: file_path
         for file_path in Path(EXTERNAL_TESTS_DIR).iterdir()
-        if file_path.is_file() and file_path.suffix == ".sh"
+        if file_path.is_file() and file_path.suffix in (".sh", ".py")
     }