Merge pull request #13873 from ethereum/foundry-prbmath-external-test

Foundry prbmath external test

.circleci/config.yml

@@ -691,7 +691,7 @@ defaults:
       name: t_native_test_ext_prb_math
       project: prb-math
       binary_type: native
-      image: cimg/node:18.16
+      image: cimg/rust:1.70
 
   - job_native_test_ext_elementfi: &job_native_test_ext_elementfi
       <<: *requires_b_ubu_static
@@ -1724,11 +1724,13 @@ workflows:
             - t_native_test_ext_yield_liquidator
             - t_native_test_ext_perpetual_pools
             - t_native_test_ext_uniswap
-            - t_native_test_ext_prb_math
             - t_native_test_ext_elementfi
             - t_native_test_ext_brink
             # NOTE: We are disabling gp2 tests due to constant failures.
             #- t_native_test_ext_gp2
+            # TODO: Dropping prb-math from the benchmarks since it is not implemented yet
+            # in the new Foundry external testing infrastructure.
+            # - t_native_test_ext_prb_math
             # NOTE: The external tests below were commented because they
             # depend on a specific version of hardhat which does not support shanghai EVM.
             #- t_native_test_ext_trident

scripts/externalTests/runners/base.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import subprocess
+from abc import ABCMeta
+from abc import abstractmethod
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+from shutil import rmtree
+from tempfile import mkdtemp
+from textwrap import dedent
+from typing import List
+from typing import Set
+
+from test_helpers import download_project
+from test_helpers import get_solc_short_version
+from test_helpers import parse_command_line
+from test_helpers import parse_custom_presets
+from test_helpers import parse_solc_version
+from test_helpers import replace_version_pragmas
+from test_helpers import settings_from_preset
+from test_helpers import SettingsPreset
+
+CURRENT_EVM_VERSION: str = "shanghai"
+
+@dataclass
+class TestConfig:
+    name: str
+    repo_url: str
+    ref_type: str
+    ref: str
+    compile_only_presets: List[SettingsPreset] = field(default_factory=list)
+    settings_presets: List[SettingsPreset] = field(default_factory=lambda: list(SettingsPreset))
+    evm_version: str = field(default=CURRENT_EVM_VERSION)
+
+    def selected_presets(self) -> Set[SettingsPreset]:
+        return set(self.compile_only_presets + self.settings_presets)
+
+
+class BaseRunner(metaclass=ABCMeta):
+    config: TestConfig
+    solc_binary_type: str
+    solc_binary_path: Path
+    presets: Set[SettingsPreset]
+
+    def __init__(self, argv, config: TestConfig):
+        args = parse_command_line(f"{config.name} external tests", argv)
+        self.config = config
+        self.solc_binary_type = args.solc_binary_type
+        self.solc_binary_path = args.solc_binary_path
+        self.presets = parse_custom_presets(args.selected_presets) if args.selected_presets else config.selected_presets()
+        self.env = os.environ.copy()
+        self.tmp_dir = mkdtemp(prefix=f"ext-test-{config.name}-")
+        self.test_dir = Path(self.tmp_dir) / "ext"
+
+    def setup_solc(self) -> str:
+        if self.solc_binary_type == "solcjs":
+            # TODO: add support to solc-js
+            raise NotImplementedError()
+        print("Setting up solc...")
+        solc_version_output = subprocess.check_output(
+            [self.solc_binary_path, "--version"],
+            shell=False,
+            encoding="utf-8"
+        ).split(":")[1]
+        return parse_solc_version(solc_version_output)
+
+    @staticmethod
+    def enter_test_dir(fn):
+        """Run a function inside the test directory"""
+
+        previous_dir = os.getcwd()
+        def f(self, *args, **kwargs):
+            try:
+                assert self.test_dir is not None
+                os.chdir(self.test_dir)
+                return fn(self, *args, **kwargs)
+            finally:
+                # Restore the previous directory after execute fn
+                os.chdir(previous_dir)
+        return f
+
+    def setup_environment(self):
+        """Configure the project build environment"""
+        print("Configuring Runner building environment...")
+        replace_version_pragmas(self.test_dir)
+
+    @enter_test_dir
+    def clean(self):
+        """Clean temporary directories"""
+        rmtree(self.tmp_dir)
+
+    @enter_test_dir
+    @abstractmethod
+    def configure(self):
+        raise NotImplementedError()
+
+    @enter_test_dir
+    @abstractmethod
+    def compile(self, preset: SettingsPreset):
+        raise NotImplementedError()
+
+    @enter_test_dir
+    @abstractmethod
+    def run_test(self):
+        raise NotImplementedError()
+
+def run_test(runner: BaseRunner):
+    print(f"Testing {runner.config.name}...\n===========================")
+    print(f"Selected settings presets: {' '.join(p.value for p in runner.presets)}")
+
+    # Configure solc compiler
+    solc_version = runner.setup_solc()
+    print(f"Using compiler version {solc_version}")
+
+    # Download project
+    download_project(runner.test_dir, runner.config.repo_url, runner.config.ref_type, runner.config.ref)
+
+    # Configure run environment
+    runner.setup_environment()
+
+    # Configure TestRunner instance
+    print(dedent(f"""\
+        Configuring runner's profiles with:
+        -------------------------------------
+        Binary type: {runner.solc_binary_type}
+        Compiler path: {runner.solc_binary_path}
+        -------------------------------------
+    """))
+    runner.configure()
+    for preset in runner.presets:
+        print("Running compile function...")
+        settings = settings_from_preset(preset, runner.config.evm_version)
+        print(dedent(f"""\
+            -------------------------------------
+            Settings preset: {preset.value}
+            Settings: {settings}
+            EVM version: {runner.config.evm_version}
+            Compiler version: {get_solc_short_version(solc_version)}
+            Compiler version (full): {solc_version}
+            -------------------------------------
+        """))
+        runner.compile(preset)
+        # TODO: COMPILE_ONLY should be a command-line option
+        if os.environ.get("COMPILE_ONLY") == "1" or preset in runner.config.compile_only_presets:
+            print("Skipping test function...")
+        else:
+            print("Running test function...")
+            runner.run_test()
+        # TODO: store_benchmark_report
+    runner.clean()
+    print("Done.")

scripts/externalTests/runners/foundry.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import re
+import subprocess
+from shutil import which
+from textwrap import dedent
+from typing import Optional
+
+from runners.base import BaseRunner
+from test_helpers import SettingsPreset
+from test_helpers import settings_from_preset
+
+def run_forge_command(command: str, env: Optional[dict] = None):
+    subprocess.run(
+        command.split(),
+        env=env if env is not None else os.environ.copy(),
+        check=True
+    )
+
+
+class FoundryRunner(BaseRunner):
+    """Configure and run Foundry-based projects"""
+
+    FOUNDRY_CONFIG_FILE = "foundry.toml"
+
+    def setup_environment(self):
+        super().setup_environment()
+        if which("forge") is None:
+            raise RuntimeError("Forge not found.")
+
+    @staticmethod
+    def profile_name(preset: SettingsPreset):
+        """Returns foundry profile name"""
+        # Replace - or + by underscore to avoid invalid toml syntax
+        return re.sub(r"(\-|\+)+", "_", preset.value)
+
+    @staticmethod
+    def profile_section(profile_fields: dict) -> str:
+        return dedent("""\
+            [profile.{name}]
+            gas_reports = ["*"]
+            auto_detect_solc = false
+            solc = "{solc}"
+            evm_version = "{evm_version}"
+            optimizer = {optimizer}
+            via_ir = {via_ir}
+
+            [profile.{name}.optimizer_details]
+            yul = {yul}
+        """).format(**profile_fields)
+
+    @BaseRunner.enter_test_dir
+    def configure(self):
+        """Configure forge tests profiles"""
+
+        profiles = []
+        for preset in self.presets:
+            settings = settings_from_preset(preset, self.config.evm_version)
+            profiles.append(self.profile_section({
+                "name": self.profile_name(preset),
+                "solc": self.solc_binary_path,
+                "evm_version": self.config.evm_version,
+                "optimizer": str(settings["optimizer"]["enabled"]).lower(),
+                "via_ir": str(settings["viaIR"]).lower(),
+                "yul": str(settings["optimizer"]["details"]["yul"]).lower(),
+            }))
+
+        with open(
+            file=self.test_dir / self.FOUNDRY_CONFIG_FILE,
+            mode="a",
+            encoding="utf-8",
+        ) as f:
+            for profile in profiles:
+                f.write(profile)
+
+        run_forge_command("forge install", self.env)
+
+    @BaseRunner.enter_test_dir
+    def compile(self, preset: SettingsPreset):
+        """Compile project"""
+
+        # Set the Foundry profile environment variable
+        self.env.update({"FOUNDRY_PROFILE": self.profile_name(preset)})
+        run_forge_command("forge build", self.env)
+
+    @BaseRunner.enter_test_dir
+    def run_test(self):
+        """Run project tests"""
+
+        run_forge_command("forge test --gas-report", self.env)

scripts/externalTests/test_helpers.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import os
+import re
+import subprocess
+import sys
+from argparse import ArgumentParser
+from enum import Enum
+from pathlib import Path
+from typing import List
+from typing import Set
+
+# Our scripts/ is not a proper Python package so we need to modify PYTHONPATH to import from it
+# pragma pylint: disable=import-error,wrong-import-position
+PROJECT_ROOT = Path(__file__).parents[2]
+sys.path.insert(0, f"{PROJECT_ROOT}/scripts/common")
+
+from git_helpers import git_commit_hash
+
+SOLC_FULL_VERSION_REGEX = re.compile(r"^[a-zA-Z: ]*(.*)$")
+SOLC_SHORT_VERSION_REGEX = re.compile(r"^([0-9.]+).*\+|\-$")
+
+
+class SettingsPreset(Enum):
+    LEGACY_NO_OPTIMIZE = 'legacy-no-optimize'
+    IR_NO_OPTIMIZE = 'ir-no-optimize'
+    LEGACY_OPTIMIZE_EVM_ONLY = 'legacy-optimize-evm-only'
+    IR_OPTIMIZE_EVM_ONLY = 'ir-optimize-evm-only'
+    LEGACY_OPTIMIZE_EVM_YUL = 'legacy-optimize-evm+yul'
+    IR_OPTIMIZE_EVM_YUL = 'ir-optimize-evm+yul'
+
+
+def compiler_settings(evm_version: str, via_ir: bool = False, optimizer: bool = False, yul: bool = False) -> dict:
+    return {
+        "optimizer": {"enabled": optimizer, "details": {"yul": yul}},
+        "evmVersion": evm_version,
+        "viaIR": via_ir,
+    }
+
+
+def settings_from_preset(preset: SettingsPreset, evm_version: str) -> dict:
+    return {
+        SettingsPreset.LEGACY_NO_OPTIMIZE:       compiler_settings(evm_version),
+        SettingsPreset.IR_NO_OPTIMIZE:           compiler_settings(evm_version, via_ir=True),
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_ONLY: compiler_settings(evm_version, optimizer=True),
+        SettingsPreset.IR_OPTIMIZE_EVM_ONLY:     compiler_settings(evm_version, via_ir=True, optimizer=True),
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_YUL:  compiler_settings(evm_version, optimizer=True, yul=True),
+        SettingsPreset.IR_OPTIMIZE_EVM_YUL:      compiler_settings(evm_version, via_ir=True, optimizer=True, yul=True),
+    }[preset]
+
+
+def parse_custom_presets(presets: List[str]) -> Set[SettingsPreset]:
+    return {SettingsPreset(p) for p in presets}
+
+def parse_command_line(description: str, args: List[str]):
+    arg_parser = ArgumentParser(description)
+    arg_parser.add_argument(
+        "solc_binary_type",
+        metavar="solc-binary-type",
+        type=str,
+        default="native",
+        choices=["native", "solcjs"],
+        help="""Solidity compiler binary type""",
+    )
+    arg_parser.add_argument(
+        "solc_binary_path",
+        metavar="solc-binary-path",
+        type=Path,
+        help="""Path to solc binary""",
+    )
+    arg_parser.add_argument(
+        "selected_presets",
+        metavar="selected-presets",
+        help="""List of compiler settings presets""",
+        nargs='*',
+    )
+    return arg_parser.parse_args(args)
+
+
+def download_project(test_dir: Path, repo_url: str, ref_type: str = "branch", ref: str = "master"):
+    assert ref_type in ("commit", "branch", "tag")
+
+    print(f"Cloning {ref_type} {ref} of {repo_url}...")
+    if ref_type == "commit":
+        os.mkdir(test_dir)
+        os.chdir(test_dir)
+        subprocess.run(["git", "init"], check=True)
+        subprocess.run(["git", "remote", "add", "origin", repo_url], check=True)
+        subprocess.run(["git", "fetch", "--depth", "1", "origin", ref], check=True)
+        subprocess.run(["git", "reset", "--hard", "FETCH_HEAD"], check=True)
+    else:
+        os.chdir(test_dir.parent)
+        subprocess.run(["git", "clone", "--no-progress", "--depth", "1", repo_url, "-b", ref, test_dir.resolve()], check=True)
+        if not test_dir.exists():
+            raise RuntimeError("Failed to clone the project.")
+        os.chdir(test_dir)
+
+    if (test_dir / ".gitmodules").exists():
+        subprocess.run(["git", "submodule", "update", "--init"], check=True)
+
+    print(f"Current commit hash: {git_commit_hash()}")
+
+
+def parse_solc_version(solc_version_string: str) -> str:
+    solc_version_match = re.search(SOLC_FULL_VERSION_REGEX, solc_version_string)
+    if solc_version_match is None:
+        raise RuntimeError(f"Solc version could not be found in: {solc_version_string}.")
+    return solc_version_match.group(1)
+
+
+def get_solc_short_version(solc_full_version: str) -> str:
+    solc_short_version_match = re.search(SOLC_SHORT_VERSION_REGEX, solc_full_version)
+    if solc_short_version_match is None:
+        raise RuntimeError(f"Error extracting short version string from: {solc_full_version}.")
+    return solc_short_version_match.group(1)
+
+
+def store_benchmark_report(self):
+    raise NotImplementedError()
+
+
+def replace_version_pragmas(test_dir: Path):
+    """
+    Replace fixed-version pragmas (part of Consensys best practice).
+    Include all directories to also cover node dependencies.
+    """
+    print("Replacing fixed-version pragmas...")
+    for source in test_dir.glob("**/*.sol"):
+        content = source.read_text(encoding="utf-8")
+        content = re.sub(r"pragma solidity [^;]+;", r"pragma solidity >=0.0;", content)
+        with open(source, "w", encoding="utf-8") as f:
+            f.write(content)

test/externalTests/prb-math.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+import sys
+from pathlib import Path
+
+# Our scripts/ is not a proper Python package so we need to modify PYTHONPATH to import from it
+# pragma pylint: disable=import-error,wrong-import-position
+PROJECT_ROOT = Path(__file__).parents[2]
+sys.path.insert(0, f"{PROJECT_ROOT}/scripts/externalTests")
+
+from runners.base import run_test
+from runners.base import TestConfig
+from runners.foundry import FoundryRunner
+from test_helpers import SettingsPreset
+
+test_config = TestConfig(
+    name="PRBMath",
+    repo_url="https://github.com/PaulRBerg/prb-math.git",
+    ref_type="branch",
+    ref="main",
+    compile_only_presets=[
+        # pylint: disable=line-too-long
+        # SettingsPreset.IR_NO_OPTIMIZE,       # Error: Yul exception:Variable expr_15699_address is 2 slot(s) too deep inside the stack. Stack too deep.
+        # SettingsPreset.IR_OPTIMIZE_EVM_ONLY, # Error: Yul exception:Variable expr_15699_address is 2 slot(s) too deep inside the stack. Stack too deep.
+    ],
+    settings_presets=[
+        SettingsPreset.LEGACY_NO_OPTIMIZE,
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_ONLY,
+        SettingsPreset.LEGACY_OPTIMIZE_EVM_YUL,
+        SettingsPreset.IR_OPTIMIZE_EVM_YUL,
+    ],
+)
+
+sys.exit(run_test(FoundryRunner(argv=sys.argv[1:], config=test_config)))

test/externalTests/prb-math.sh

@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-
-# ------------------------------------------------------------------------------
-# This file is part of solidity.
-#
-# solidity is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# solidity is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with solidity.  If not, see <http://www.gnu.org/licenses/>
-#
-# (c) 2022 solidity contributors.
-#------------------------------------------------------------------------------
-
-set -e
-
-source scripts/common.sh
-source scripts/externalTests/common.sh
-
-REPO_ROOT=$(realpath "$(dirname "$0")/../..")
-
-verify_input "$@"
-BINARY_TYPE="$1"
-BINARY_PATH="$(realpath "$2")"
-SELECTED_PRESETS="$3"
-
-function compile_fn { yarn compile; }
-# NOTE: `yarn test` runs `mocha` which seems to disable the gas reporter.
-function test_fn { npx --no hardhat --no-compile test; }
-
-function prb_math_test
-{
-    local repo="https://github.com/paulrberg/prb-math"
-    local ref_type=branch
-    # We currently pin the prb-math version to the latest version that support hardhat
-    # Please see here for details: https://github.com/ethereum/solidity/issues/13767
-    local ref=v2.5.0
-    local config_file="hardhat.config.ts"
-    local config_var="config"
-
-    local compile_only_presets=(
-        ir-no-optimize            # Tests fail with "Error: Transaction reverted: trying to deploy a contract whose code is too large"
-    )
-    local settings_presets=(
-        "${compile_only_presets[@]}"
-        ir-optimize-evm-only
-        ir-optimize-evm+yul
-        legacy-optimize-evm-only
-        legacy-optimize-evm+yul
-        legacy-no-optimize
-    )
-
-    [[ $SELECTED_PRESETS != "" ]] || SELECTED_PRESETS=$(circleci_select_steps_multiarg "${settings_presets[@]}")
-    print_presets_or_exit "$SELECTED_PRESETS"
-
-    setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
-    download_project "$repo" "$ref_type" "$ref" "$DIR"
-
-    cp .env.example .env
-
-    # The project has yarn 3.1.0 binary stored in the repo and yarnrc forces the yarn 1.x binary
-    # installed system-wide to use it. Unfortunately Yarn 3 fails in weird ways when we remove
-    # yarn.lock. Remove the config to restore Yarn 1.x.
-    rm .yarnrc.yml
-
-    # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables
-    # them for other presets but that's fine - we want same code run for benchmarks to be comparable.
-    # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/3365).
-    pushd test/contracts/prbMathUd60x18/pure/
-    sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts
-    sed -i 's|context(\("when the sum does not overflow"\)|context.skip(\1|g' add.test.ts
-    sed -i 's|context(\("when both operands are zero"\)|context.skip(\1|g' avg.test.ts
-    sed -i 's|context(\("when one operand is zero and the other is not zero"\)|context.skip(\1|g' avg.test.ts
-    sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts
-    sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts
-    popd
-    pushd test/contracts/prbMathSd59x18/pure/
-    sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts
-    sed -i 's|context(\("when the sum underflows"\)|context.skip(\1|g' add.test.ts
-    sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts
-    sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts
-    sed -i 's|context(\("when the difference underflows"\)|context.skip(\1|g' sub.test.ts
-    sed -i 's|context(\("when the difference overflows"\)|context.skip(\1|g' sub.test.ts
-    popd
-
-    neutralize_package_lock
-    neutralize_package_json_hooks
-    force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"
-    force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")" "$config_var"
-    yarn install --no-lock-file
-    yarn add hardhat-gas-reporter
-
-    # Workaround for error caused by the last release of hardhat-waffle@2.0.6 that bumps ethereum-waffle
-    # to version 4.0.10 and breaks prb-math build with the following error:
-    #
-    #  Cannot find module 'ethereum-waffle/dist/cjs/src/deployContract'
-    #
-    # See: https://github.com/NomicFoundation/hardhat-waffle/commit/83ee9cb36ee59d0bedacbbd00043f030af104ad0
-    yarn add '@nomiclabs/hardhat-waffle@2.0.5'
-
-    replace_version_pragmas
-
-    for preset in $SELECTED_PRESETS; do
-        hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
-        store_benchmark_report hardhat prb-math "$repo" "$preset"
-    done
-}
-
-external_test PRBMath prb_math_test

test/external_tests.py

@@ -36,7 +36,7 @@ def detect_external_tests() -> dict:
     return {
         file_path.stem: file_path
         for file_path in Path(EXTERNAL_TESTS_DIR).iterdir()
-        if file_path.is_file() and file_path.suffix == ".sh"
+        if file_path.is_file() and file_path.suffix in (".sh", ".py")
     }