diff --git a/.circleci/config.yml b/.circleci/config.yml index bb6e562f1..38e9ee47f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -691,7 +691,7 @@ defaults: name: t_native_test_ext_prb_math project: prb-math binary_type: native - image: cimg/node:18.16 + image: cimg/rust:1.70 - job_native_test_ext_elementfi: &job_native_test_ext_elementfi <<: *requires_b_ubu_static @@ -1724,11 +1724,13 @@ workflows: - t_native_test_ext_yield_liquidator - t_native_test_ext_perpetual_pools - t_native_test_ext_uniswap - - t_native_test_ext_prb_math - t_native_test_ext_elementfi - t_native_test_ext_brink # NOTE: We are disabling gp2 tests due to constant failures. #- t_native_test_ext_gp2 + # TODO: Dropping prb-math from the benchmarks since it is not implemented yet + # in the new Foundry external testing infrastructure. + # - t_native_test_ext_prb_math # NOTE: The external tests below were commented because they # depend on a specific version of hardhat which does not support shanghai EVM. #- t_native_test_ext_trident diff --git a/scripts/externalTests/runners/base.py b/scripts/externalTests/runners/base.py new file mode 100644 index 000000000..a79ed5344 --- /dev/null +++ b/scripts/externalTests/runners/base.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2023 solidity contributors. +# ------------------------------------------------------------------------------ + +import os +import subprocess +from abc import ABCMeta +from abc import abstractmethod +from dataclasses import dataclass +from dataclasses import field +from pathlib import Path +from shutil import rmtree +from tempfile import mkdtemp +from textwrap import dedent +from typing import List +from typing import Set + +from test_helpers import download_project +from test_helpers import get_solc_short_version +from test_helpers import parse_command_line +from test_helpers import parse_custom_presets +from test_helpers import parse_solc_version +from test_helpers import replace_version_pragmas +from test_helpers import settings_from_preset +from test_helpers import SettingsPreset + +CURRENT_EVM_VERSION: str = "shanghai" + +@dataclass +class TestConfig: + name: str + repo_url: str + ref_type: str + ref: str + compile_only_presets: List[SettingsPreset] = field(default_factory=list) + settings_presets: List[SettingsPreset] = field(default_factory=lambda: list(SettingsPreset)) + evm_version: str = field(default=CURRENT_EVM_VERSION) + + def selected_presets(self) -> Set[SettingsPreset]: + return set(self.compile_only_presets + self.settings_presets) + + +class BaseRunner(metaclass=ABCMeta): + config: TestConfig + solc_binary_type: str + solc_binary_path: Path + presets: Set[SettingsPreset] + + def __init__(self, argv, config: TestConfig): + args = parse_command_line(f"{config.name} external tests", argv) + self.config = config + self.solc_binary_type = args.solc_binary_type + self.solc_binary_path = args.solc_binary_path + self.presets = parse_custom_presets(args.selected_presets) if args.selected_presets else config.selected_presets() + self.env = os.environ.copy() + self.tmp_dir = mkdtemp(prefix=f"ext-test-{config.name}-") + self.test_dir = Path(self.tmp_dir) / "ext" + + def setup_solc(self) -> str: + if self.solc_binary_type == "solcjs": + # TODO: add support to solc-js + raise NotImplementedError() + print("Setting up solc...") + solc_version_output = subprocess.check_output( + [self.solc_binary_path, "--version"], + shell=False, + encoding="utf-8" + ).split(":")[1] + return parse_solc_version(solc_version_output) + + @staticmethod + def enter_test_dir(fn): + """Run a function inside the test directory""" + + previous_dir = os.getcwd() + def f(self, *args, **kwargs): + try: + assert self.test_dir is not None + os.chdir(self.test_dir) + return fn(self, *args, **kwargs) + finally: + # Restore the previous directory after execute fn + os.chdir(previous_dir) + return f + + def setup_environment(self): + """Configure the project build environment""" + print("Configuring Runner building environment...") + replace_version_pragmas(self.test_dir) + + @enter_test_dir + def clean(self): + """Clean temporary directories""" + rmtree(self.tmp_dir) + + @enter_test_dir + @abstractmethod + def configure(self): + raise NotImplementedError() + + @enter_test_dir + @abstractmethod + def compile(self, preset: SettingsPreset): + raise NotImplementedError() + + @enter_test_dir + @abstractmethod + def run_test(self): + raise NotImplementedError() + +def run_test(runner: BaseRunner): + print(f"Testing {runner.config.name}...\n===========================") + print(f"Selected settings presets: {' '.join(p.value for p in runner.presets)}") + + # Configure solc compiler + solc_version = runner.setup_solc() + print(f"Using compiler version {solc_version}") + + # Download project + download_project(runner.test_dir, runner.config.repo_url, runner.config.ref_type, runner.config.ref) + + # Configure run environment + runner.setup_environment() + + # Configure TestRunner instance + print(dedent(f"""\ + Configuring runner's profiles with: + ------------------------------------- + Binary type: {runner.solc_binary_type} + Compiler path: {runner.solc_binary_path} + ------------------------------------- + """)) + runner.configure() + for preset in runner.presets: + print("Running compile function...") + settings = settings_from_preset(preset, runner.config.evm_version) + print(dedent(f"""\ + ------------------------------------- + Settings preset: {preset.value} + Settings: {settings} + EVM version: {runner.config.evm_version} + Compiler version: {get_solc_short_version(solc_version)} + Compiler version (full): {solc_version} + ------------------------------------- + """)) + runner.compile(preset) + # TODO: COMPILE_ONLY should be a command-line option + if os.environ.get("COMPILE_ONLY") == "1" or preset in runner.config.compile_only_presets: + print("Skipping test function...") + else: + print("Running test function...") + runner.run_test() + # TODO: store_benchmark_report + runner.clean() + print("Done.") diff --git a/scripts/externalTests/runners/foundry.py b/scripts/externalTests/runners/foundry.py new file mode 100644 index 000000000..0b171250f --- /dev/null +++ b/scripts/externalTests/runners/foundry.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2023 solidity contributors. +# ------------------------------------------------------------------------------ + +import os +import re +import subprocess +from shutil import which +from textwrap import dedent +from typing import Optional + +from runners.base import BaseRunner +from test_helpers import SettingsPreset +from test_helpers import settings_from_preset + +def run_forge_command(command: str, env: Optional[dict] = None): + subprocess.run( + command.split(), + env=env if env is not None else os.environ.copy(), + check=True + ) + + +class FoundryRunner(BaseRunner): + """Configure and run Foundry-based projects""" + + FOUNDRY_CONFIG_FILE = "foundry.toml" + + def setup_environment(self): + super().setup_environment() + if which("forge") is None: + raise RuntimeError("Forge not found.") + + @staticmethod + def profile_name(preset: SettingsPreset): + """Returns foundry profile name""" + # Replace - or + by underscore to avoid invalid toml syntax + return re.sub(r"(\-|\+)+", "_", preset.value) + + @staticmethod + def profile_section(profile_fields: dict) -> str: + return dedent("""\ + [profile.{name}] + gas_reports = ["*"] + auto_detect_solc = false + solc = "{solc}" + evm_version = "{evm_version}" + optimizer = {optimizer} + via_ir = {via_ir} + + [profile.{name}.optimizer_details] + yul = {yul} + """).format(**profile_fields) + + @BaseRunner.enter_test_dir + def configure(self): + """Configure forge tests profiles""" + + profiles = [] + for preset in self.presets: + settings = settings_from_preset(preset, self.config.evm_version) + profiles.append(self.profile_section({ + "name": self.profile_name(preset), + "solc": self.solc_binary_path, + "evm_version": self.config.evm_version, + "optimizer": str(settings["optimizer"]["enabled"]).lower(), + "via_ir": str(settings["viaIR"]).lower(), + "yul": str(settings["optimizer"]["details"]["yul"]).lower(), + })) + + with open( + file=self.test_dir / self.FOUNDRY_CONFIG_FILE, + mode="a", + encoding="utf-8", + ) as f: + for profile in profiles: + f.write(profile) + + run_forge_command("forge install", self.env) + + @BaseRunner.enter_test_dir + def compile(self, preset: SettingsPreset): + """Compile project""" + + # Set the Foundry profile environment variable + self.env.update({"FOUNDRY_PROFILE": self.profile_name(preset)}) + run_forge_command("forge build", self.env) + + @BaseRunner.enter_test_dir + def run_test(self): + """Run project tests""" + + run_forge_command("forge test --gas-report", self.env) diff --git a/scripts/externalTests/test_helpers.py b/scripts/externalTests/test_helpers.py new file mode 100644 index 000000000..9a573d50a --- /dev/null +++ b/scripts/externalTests/test_helpers.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2023 solidity contributors. +# ------------------------------------------------------------------------------ + +import os +import re +import subprocess +import sys +from argparse import ArgumentParser +from enum import Enum +from pathlib import Path +from typing import List +from typing import Set + +# Our scripts/ is not a proper Python package so we need to modify PYTHONPATH to import from it +# pragma pylint: disable=import-error,wrong-import-position +PROJECT_ROOT = Path(__file__).parents[2] +sys.path.insert(0, f"{PROJECT_ROOT}/scripts/common") + +from git_helpers import git_commit_hash + +SOLC_FULL_VERSION_REGEX = re.compile(r"^[a-zA-Z: ]*(.*)$") +SOLC_SHORT_VERSION_REGEX = re.compile(r"^([0-9.]+).*\+|\-$") + + +class SettingsPreset(Enum): + LEGACY_NO_OPTIMIZE = 'legacy-no-optimize' + IR_NO_OPTIMIZE = 'ir-no-optimize' + LEGACY_OPTIMIZE_EVM_ONLY = 'legacy-optimize-evm-only' + IR_OPTIMIZE_EVM_ONLY = 'ir-optimize-evm-only' + LEGACY_OPTIMIZE_EVM_YUL = 'legacy-optimize-evm+yul' + IR_OPTIMIZE_EVM_YUL = 'ir-optimize-evm+yul' + + +def compiler_settings(evm_version: str, via_ir: bool = False, optimizer: bool = False, yul: bool = False) -> dict: + return { + "optimizer": {"enabled": optimizer, "details": {"yul": yul}}, + "evmVersion": evm_version, + "viaIR": via_ir, + } + + +def settings_from_preset(preset: SettingsPreset, evm_version: str) -> dict: + return { + SettingsPreset.LEGACY_NO_OPTIMIZE: compiler_settings(evm_version), + SettingsPreset.IR_NO_OPTIMIZE: compiler_settings(evm_version, via_ir=True), + SettingsPreset.LEGACY_OPTIMIZE_EVM_ONLY: compiler_settings(evm_version, optimizer=True), + SettingsPreset.IR_OPTIMIZE_EVM_ONLY: compiler_settings(evm_version, via_ir=True, optimizer=True), + SettingsPreset.LEGACY_OPTIMIZE_EVM_YUL: compiler_settings(evm_version, optimizer=True, yul=True), + SettingsPreset.IR_OPTIMIZE_EVM_YUL: compiler_settings(evm_version, via_ir=True, optimizer=True, yul=True), + }[preset] + + +def parse_custom_presets(presets: List[str]) -> Set[SettingsPreset]: + return {SettingsPreset(p) for p in presets} + +def parse_command_line(description: str, args: List[str]): + arg_parser = ArgumentParser(description) + arg_parser.add_argument( + "solc_binary_type", + metavar="solc-binary-type", + type=str, + default="native", + choices=["native", "solcjs"], + help="""Solidity compiler binary type""", + ) + arg_parser.add_argument( + "solc_binary_path", + metavar="solc-binary-path", + type=Path, + help="""Path to solc binary""", + ) + arg_parser.add_argument( + "selected_presets", + metavar="selected-presets", + help="""List of compiler settings presets""", + nargs='*', + ) + return arg_parser.parse_args(args) + + +def download_project(test_dir: Path, repo_url: str, ref_type: str = "branch", ref: str = "master"): + assert ref_type in ("commit", "branch", "tag") + + print(f"Cloning {ref_type} {ref} of {repo_url}...") + if ref_type == "commit": + os.mkdir(test_dir) + os.chdir(test_dir) + subprocess.run(["git", "init"], check=True) + subprocess.run(["git", "remote", "add", "origin", repo_url], check=True) + subprocess.run(["git", "fetch", "--depth", "1", "origin", ref], check=True) + subprocess.run(["git", "reset", "--hard", "FETCH_HEAD"], check=True) + else: + os.chdir(test_dir.parent) + subprocess.run(["git", "clone", "--no-progress", "--depth", "1", repo_url, "-b", ref, test_dir.resolve()], check=True) + if not test_dir.exists(): + raise RuntimeError("Failed to clone the project.") + os.chdir(test_dir) + + if (test_dir / ".gitmodules").exists(): + subprocess.run(["git", "submodule", "update", "--init"], check=True) + + print(f"Current commit hash: {git_commit_hash()}") + + +def parse_solc_version(solc_version_string: str) -> str: + solc_version_match = re.search(SOLC_FULL_VERSION_REGEX, solc_version_string) + if solc_version_match is None: + raise RuntimeError(f"Solc version could not be found in: {solc_version_string}.") + return solc_version_match.group(1) + + +def get_solc_short_version(solc_full_version: str) -> str: + solc_short_version_match = re.search(SOLC_SHORT_VERSION_REGEX, solc_full_version) + if solc_short_version_match is None: + raise RuntimeError(f"Error extracting short version string from: {solc_full_version}.") + return solc_short_version_match.group(1) + + +def store_benchmark_report(self): + raise NotImplementedError() + + +def replace_version_pragmas(test_dir: Path): + """ + Replace fixed-version pragmas (part of Consensys best practice). + Include all directories to also cover node dependencies. + """ + print("Replacing fixed-version pragmas...") + for source in test_dir.glob("**/*.sol"): + content = source.read_text(encoding="utf-8") + content = re.sub(r"pragma solidity [^;]+;", r"pragma solidity >=0.0;", content) + with open(source, "w", encoding="utf-8") as f: + f.write(content) diff --git a/test/externalTests/prb-math.py b/test/externalTests/prb-math.py new file mode 100755 index 000000000..0a4ce8ff9 --- /dev/null +++ b/test/externalTests/prb-math.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2023 solidity contributors. +# ------------------------------------------------------------------------------ + +import sys +from pathlib import Path + +# Our scripts/ is not a proper Python package so we need to modify PYTHONPATH to import from it +# pragma pylint: disable=import-error,wrong-import-position +PROJECT_ROOT = Path(__file__).parents[2] +sys.path.insert(0, f"{PROJECT_ROOT}/scripts/externalTests") + +from runners.base import run_test +from runners.base import TestConfig +from runners.foundry import FoundryRunner +from test_helpers import SettingsPreset + +test_config = TestConfig( + name="PRBMath", + repo_url="https://github.com/PaulRBerg/prb-math.git", + ref_type="branch", + ref="main", + compile_only_presets=[ + # pylint: disable=line-too-long + # SettingsPreset.IR_NO_OPTIMIZE, # Error: Yul exception:Variable expr_15699_address is 2 slot(s) too deep inside the stack. Stack too deep. + # SettingsPreset.IR_OPTIMIZE_EVM_ONLY, # Error: Yul exception:Variable expr_15699_address is 2 slot(s) too deep inside the stack. Stack too deep. + ], + settings_presets=[ + SettingsPreset.LEGACY_NO_OPTIMIZE, + SettingsPreset.LEGACY_OPTIMIZE_EVM_ONLY, + SettingsPreset.LEGACY_OPTIMIZE_EVM_YUL, + SettingsPreset.IR_OPTIMIZE_EVM_YUL, + ], +) + +sys.exit(run_test(FoundryRunner(argv=sys.argv[1:], config=test_config))) diff --git a/test/externalTests/prb-math.sh b/test/externalTests/prb-math.sh deleted file mode 100755 index fcf1aa208..000000000 --- a/test/externalTests/prb-math.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -# ------------------------------------------------------------------------------ -# This file is part of solidity. -# -# solidity is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# solidity is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with solidity. If not, see -# -# (c) 2022 solidity contributors. -#------------------------------------------------------------------------------ - -set -e - -source scripts/common.sh -source scripts/externalTests/common.sh - -REPO_ROOT=$(realpath "$(dirname "$0")/../..") - -verify_input "$@" -BINARY_TYPE="$1" -BINARY_PATH="$(realpath "$2")" -SELECTED_PRESETS="$3" - -function compile_fn { yarn compile; } -# NOTE: `yarn test` runs `mocha` which seems to disable the gas reporter. -function test_fn { npx --no hardhat --no-compile test; } - -function prb_math_test -{ - local repo="https://github.com/paulrberg/prb-math" - local ref_type=branch - # We currently pin the prb-math version to the latest version that support hardhat - # Please see here for details: https://github.com/ethereum/solidity/issues/13767 - local ref=v2.5.0 - local config_file="hardhat.config.ts" - local config_var="config" - - local compile_only_presets=( - ir-no-optimize # Tests fail with "Error: Transaction reverted: trying to deploy a contract whose code is too large" - ) - local settings_presets=( - "${compile_only_presets[@]}" - ir-optimize-evm-only - ir-optimize-evm+yul - legacy-optimize-evm-only - legacy-optimize-evm+yul - legacy-no-optimize - ) - - [[ $SELECTED_PRESETS != "" ]] || SELECTED_PRESETS=$(circleci_select_steps_multiarg "${settings_presets[@]}") - print_presets_or_exit "$SELECTED_PRESETS" - - setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" - download_project "$repo" "$ref_type" "$ref" "$DIR" - - cp .env.example .env - - # The project has yarn 3.1.0 binary stored in the repo and yarnrc forces the yarn 1.x binary - # installed system-wide to use it. Unfortunately Yarn 3 fails in weird ways when we remove - # yarn.lock. Remove the config to restore Yarn 1.x. - rm .yarnrc.yml - - # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables - # them for other presets but that's fine - we want same code run for benchmarks to be comparable. - # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/3365). - pushd test/contracts/prbMathUd60x18/pure/ - sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts - sed -i 's|context(\("when the sum does not overflow"\)|context.skip(\1|g' add.test.ts - sed -i 's|context(\("when both operands are zero"\)|context.skip(\1|g' avg.test.ts - sed -i 's|context(\("when one operand is zero and the other is not zero"\)|context.skip(\1|g' avg.test.ts - sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts - sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts - popd - pushd test/contracts/prbMathSd59x18/pure/ - sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts - sed -i 's|context(\("when the sum underflows"\)|context.skip(\1|g' add.test.ts - sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts - sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts - sed -i 's|context(\("when the difference underflows"\)|context.skip(\1|g' sub.test.ts - sed -i 's|context(\("when the difference overflows"\)|context.skip(\1|g' sub.test.ts - popd - - neutralize_package_lock - neutralize_package_json_hooks - force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" - force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")" "$config_var" - yarn install --no-lock-file - yarn add hardhat-gas-reporter - - # Workaround for error caused by the last release of hardhat-waffle@2.0.6 that bumps ethereum-waffle - # to version 4.0.10 and breaks prb-math build with the following error: - # - # Cannot find module 'ethereum-waffle/dist/cjs/src/deployContract' - # - # See: https://github.com/NomicFoundation/hardhat-waffle/commit/83ee9cb36ee59d0bedacbbd00043f030af104ad0 - yarn add '@nomiclabs/hardhat-waffle@2.0.5' - - replace_version_pragmas - - for preset in $SELECTED_PRESETS; do - hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" - store_benchmark_report hardhat prb-math "$repo" "$preset" - done -} - -external_test PRBMath prb_math_test diff --git a/test/external_tests.py b/test/external_tests.py index 59b22af7e..a7b8e87e8 100755 --- a/test/external_tests.py +++ b/test/external_tests.py @@ -36,7 +36,7 @@ def detect_external_tests() -> dict: return { file_path.stem: file_path for file_path in Path(EXTERNAL_TESTS_DIR).iterdir() - if file_path.is_file() and file_path.suffix == ".sh" + if file_path.is_file() and file_path.suffix in (".sh", ".py") }