solidity/scripts/externalTests/runners/base.py
r0qs ab4f5f2983
Initial prototype of prb-math external tests using foundry rewritten in python
Co-authored-by: Kamil Śliwak <kamil.sliwak@codepoets.it>
2023-07-21 20:25:58 +02:00

173 lines
5.9 KiB
Python

#!/usr/bin/env python3
# ------------------------------------------------------------------------------
# This file is part of solidity.
#
# solidity is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# solidity is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with solidity. If not, see <http://www.gnu.org/licenses/>
#
# (c) 2023 solidity contributors.
# ------------------------------------------------------------------------------
import os
import subprocess
from abc import ABCMeta
from abc import abstractmethod
from dataclasses import dataclass
from dataclasses import field
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
from textwrap import dedent
from typing import List
from typing import Set
from test_helpers import download_project
from test_helpers import get_solc_short_version
from test_helpers import parse_command_line
from test_helpers import parse_custom_presets
from test_helpers import parse_solc_version
from test_helpers import replace_version_pragmas
from test_helpers import settings_from_preset
from test_helpers import SettingsPreset
CURRENT_EVM_VERSION: str = "shanghai"
@dataclass
class TestConfig:
name: str
repo_url: str
ref_type: str
ref: str
compile_only_presets: List[SettingsPreset] = field(default_factory=list)
settings_presets: List[SettingsPreset] = field(default_factory=lambda: list(SettingsPreset))
evm_version: str = field(default=CURRENT_EVM_VERSION)
def selected_presets(self) -> Set[SettingsPreset]:
return set(self.compile_only_presets + self.settings_presets)
class BaseRunner(metaclass=ABCMeta):
config: TestConfig
solc_binary_type: str
solc_binary_path: Path
presets: Set[SettingsPreset]
def __init__(self, argv, config: TestConfig):
args = parse_command_line(f"{config.name} external tests", argv)
self.config = config
self.solc_binary_type = args.solc_binary_type
self.solc_binary_path = args.solc_binary_path
self.presets = parse_custom_presets(args.selected_presets) if args.selected_presets else config.selected_presets()
self.env = os.environ.copy()
self.tmp_dir = mkdtemp(prefix=f"ext-test-{config.name}-")
self.test_dir = Path(self.tmp_dir) / "ext"
def setup_solc(self) -> str:
if self.solc_binary_type == "solcjs":
# TODO: add support to solc-js
raise NotImplementedError()
print("Setting up solc...")
solc_version_output = subprocess.check_output(
[self.solc_binary_path, "--version"],
shell=False,
encoding="utf-8"
).split(":")[1]
return parse_solc_version(solc_version_output)
@staticmethod
def enter_test_dir(fn):
"""Run a function inside the test directory"""
previous_dir = os.getcwd()
def f(self, *args, **kwargs):
try:
assert self.test_dir is not None
os.chdir(self.test_dir)
return fn(self, *args, **kwargs)
finally:
# Restore the previous directory after execute fn
os.chdir(previous_dir)
return f
def setup_environment(self):
"""Configure the project build environment"""
print("Configuring Runner building environment...")
replace_version_pragmas(self.test_dir)
@enter_test_dir
def clean(self):
"""Clean temporary directories"""
rmtree(self.tmp_dir)
@enter_test_dir
@abstractmethod
def configure(self):
raise NotImplementedError()
@enter_test_dir
@abstractmethod
def compile(self, preset: SettingsPreset):
raise NotImplementedError()
@enter_test_dir
@abstractmethod
def run_test(self):
raise NotImplementedError()
def run_test(runner: BaseRunner):
print(f"Testing {runner.config.name}...\n===========================")
print(f"Selected settings presets: {' '.join(p.value for p in runner.presets)}")
# Configure solc compiler
solc_version = runner.setup_solc()
print(f"Using compiler version {solc_version}")
# Download project
download_project(runner.test_dir, runner.config.repo_url, runner.config.ref_type, runner.config.ref)
# Configure run environment
runner.setup_environment()
# Configure TestRunner instance
print(dedent(f"""\
Configuring runner's profiles with:
-------------------------------------
Binary type: {runner.solc_binary_type}
Compiler path: {runner.solc_binary_path}
-------------------------------------
"""))
runner.configure()
for preset in runner.presets:
print("Running compile function...")
settings = settings_from_preset(preset, runner.config.evm_version)
print(dedent(f"""\
-------------------------------------
Settings preset: {preset.value}
Settings: {settings}
EVM version: {runner.config.evm_version}
Compiler version: {get_solc_short_version(solc_version)}
Compiler version (full): {solc_version}
-------------------------------------
"""))
runner.compile(preset)
# TODO: COMPILE_ONLY should be a command-line option
if os.environ.get("COMPILE_ONLY") == "1" or preset in runner.config.compile_only_presets:
print("Skipping test function...")
else:
print("Running test function...")
runner.run_test()
# TODO: store_benchmark_report
runner.clean()
print("Done.")