diff --git a/test/externalTests/common.sh b/scripts/externalTests/common.sh similarity index 100% rename from test/externalTests/common.sh rename to scripts/externalTests/common.sh diff --git a/test/externalTests.sh b/test/externalTests.sh deleted file mode 100755 index 4746e2a58..000000000 --- a/test/externalTests.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -#------------------------------------------------------------------------------ -# Bash script to run external Solidity tests. -# -# Argument: Path to soljson.js to test. -# -# Requires npm, networking access and git to download the tests. -# -# ------------------------------------------------------------------------------ -# This file is part of solidity. -# -# solidity is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# solidity is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with solidity. If not, see -# -# (c) 2016 solidity contributors. -#------------------------------------------------------------------------------ - -set -e - -source scripts/common.sh -source test/externalTests/common.sh - -REPO_ROOT=$(realpath "$(dirname "$0")/..") - -verify_input "$@" - -printTask "Running external tests..." - -"${REPO_ROOT}/test/externalTests/zeppelin.sh" "$@" -"${REPO_ROOT}/test/externalTests/gnosis.sh" "$@" -"${REPO_ROOT}/test/externalTests/colony.sh" "$@" -"${REPO_ROOT}/test/externalTests/ens.sh" "$@" -"${REPO_ROOT}/test/externalTests/trident.sh" "$@" -"${REPO_ROOT}/test/externalTests/euler.sh" "$@" -"${REPO_ROOT}/test/externalTests/yield-liquidator.sh" "$@" -"${REPO_ROOT}/test/externalTests/bleeps.sh" "$@" -"${REPO_ROOT}/test/externalTests/pool-together.sh" "$@" -"${REPO_ROOT}/test/externalTests/perpetual-pools.sh" "$@" -"${REPO_ROOT}/test/externalTests/uniswap.sh" "$@" -"${REPO_ROOT}/test/externalTests/prb-math.sh" "$@" -"${REPO_ROOT}/test/externalTests/elementfi.sh" "$@" -"${REPO_ROOT}/test/externalTests/brink.sh" "$@" -"${REPO_ROOT}/test/externalTests/chainlink.sh" "$@" -"${REPO_ROOT}/test/externalTests/gp2.sh" "$@" diff --git a/test/externalTests/bleeps.sh b/test/externalTests/bleeps.sh index 22e54ac9c..939552100 100755 --- a/test/externalTests/bleeps.sh +++ b/test/externalTests/bleeps.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/brink.sh b/test/externalTests/brink.sh index 901a0cfc0..77dd62f94 100755 --- a/test/externalTests/brink.sh +++ b/test/externalTests/brink.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/chainlink.sh b/test/externalTests/chainlink.sh index 249a1254c..a11266c3b 100755 --- a/test/externalTests/chainlink.sh +++ b/test/externalTests/chainlink.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/colony.sh b/test/externalTests/colony.sh index 8582411b9..402865107 100755 --- a/test/externalTests/colony.sh +++ b/test/externalTests/colony.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/elementfi.sh b/test/externalTests/elementfi.sh index b667a4bbc..b3348e339 100755 --- a/test/externalTests/elementfi.sh +++ b/test/externalTests/elementfi.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/ens.sh b/test/externalTests/ens.sh index 77395160d..7f713f4ab 100755 --- a/test/externalTests/ens.sh +++ b/test/externalTests/ens.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/euler.sh b/test/externalTests/euler.sh index 78e89d147..586526124 100755 --- a/test/externalTests/euler.sh +++ b/test/externalTests/euler.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/gnosis.sh b/test/externalTests/gnosis.sh index 00ea2597e..de0624c06 100755 --- a/test/externalTests/gnosis.sh +++ b/test/externalTests/gnosis.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/gp2.sh b/test/externalTests/gp2.sh index 120379478..a73f1453b 100755 --- a/test/externalTests/gp2.sh +++ b/test/externalTests/gp2.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/perpetual-pools.sh b/test/externalTests/perpetual-pools.sh index 18fb308af..70a14c499 100755 --- a/test/externalTests/perpetual-pools.sh +++ b/test/externalTests/perpetual-pools.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/pool-together.sh b/test/externalTests/pool-together.sh index 86cd3ee68..77b3ad149 100755 --- a/test/externalTests/pool-together.sh +++ b/test/externalTests/pool-together.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/prb-math.sh b/test/externalTests/prb-math.sh index 78a39519e..99c7c2716 100755 --- a/test/externalTests/prb-math.sh +++ b/test/externalTests/prb-math.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/solc-js/solc-js.sh b/test/externalTests/solc-js/solc-js.sh index bf7ca8762..af325326c 100755 --- a/test/externalTests/solc-js/solc-js.sh +++ b/test/externalTests/solc-js/solc-js.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh SOLJSON="$1" VERSION="$2" diff --git a/test/externalTests/trident.sh b/test/externalTests/trident.sh index b5d48b500..f02123217 100755 --- a/test/externalTests/trident.sh +++ b/test/externalTests/trident.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/uniswap.sh b/test/externalTests/uniswap.sh index 8b07aee84..d9a25138f 100755 --- a/test/externalTests/uniswap.sh +++ b/test/externalTests/uniswap.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/yield-liquidator.sh b/test/externalTests/yield-liquidator.sh index a22443311..8b256d797 100755 --- a/test/externalTests/yield-liquidator.sh +++ b/test/externalTests/yield-liquidator.sh @@ -22,7 +22,7 @@ set -e source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/externalTests/zeppelin.sh b/test/externalTests/zeppelin.sh index 2a95fc8dd..46b3f2d89 100755 --- a/test/externalTests/zeppelin.sh +++ b/test/externalTests/zeppelin.sh @@ -27,7 +27,7 @@ set -e export NODE_OPTIONS="--max-old-space-size=4096" source scripts/common.sh -source test/externalTests/common.sh +source scripts/externalTests/common.sh REPO_ROOT=$(realpath "$(dirname "$0")/../..") diff --git a/test/external_tests.py b/test/external_tests.py new file mode 100755 index 000000000..e25311793 --- /dev/null +++ b/test/external_tests.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +# ------------------------------------------------------------------------------ +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) 2023 solidity contributors. +# ------------------------------------------------------------------------------ + +from argparse import ArgumentParser, Namespace +import os +from pathlib import Path +import sys +import subprocess + +EXTERNAL_TESTS_DIR = Path(__file__).parent / "externalTests" + + +class ExternalTestNotFound(Exception): + pass + + +def detect_external_tests() -> dict: + return { + file_path.stem: file_path + for file_path in Path(EXTERNAL_TESTS_DIR).iterdir() + if file_path.is_file() and file_path.suffix == ".sh" + } + + +def display_available_external_tests(_): + print("Available external tests:") + print(*detect_external_tests().keys()) + + +def run_test_scripts(solc_binary_type: str, solc_binary_path: Path, tests: dict): + for test_name, test_script_path in tests.items(): + print(f"Running {test_name} external test...") + subprocess.run( + [test_script_path, solc_binary_type, solc_binary_path], + check=True + ) + + +def run_external_tests(args: dict): + solc_binary_type = args["solc_binary_type"] + solc_binary_path = args["solc_binary_path"] + + all_test_scripts = detect_external_tests() + if args["run_all"]: + run_test_scripts(solc_binary_type, solc_binary_path, all_test_scripts) + else: + selected_tests = args["selected_tests"] + if selected_tests: + unrecognized_tests = set(selected_tests) - set(all_test_scripts.keys()) + if unrecognized_tests != set(): + raise ExternalTestNotFound( + f"External test(s) not found: {', '.join(unrecognized_tests)}" + ) + run_test_scripts( + solc_binary_type, + solc_binary_path, + {k: all_test_scripts[k] for k in selected_tests}, + ) + raise ExternalTestNotFound( + "External test was not selected. Please use --run or --run-all option" + ) + + +def parse_commandline() -> Namespace: + script_description = "Script to run external Solidity tests." + + parser = ArgumentParser(description=script_description) + subparser = parser.add_subparsers() + list_command = subparser.add_parser( + "list", + help="List all available external tests.", + ) + list_command.set_defaults(cmd=display_available_external_tests) + + run_command = subparser.add_parser( + "test", + help="Run external tests.", + ) + run_command.set_defaults(cmd=run_external_tests) + + run_command.add_argument( + "--solc-binary-type", + dest="solc_binary_type", + type=str, + required=True, + choices=["native", "solcjs"], + help="Type of the solidity compiler binary to be used.", + ) + run_command.add_argument( + "--solc-binary-path", + dest="solc_binary_path", + type=Path, + required=True, + help="Path to the solidity compiler binary.", + ) + + running_mode = run_command.add_mutually_exclusive_group() + running_mode.add_argument( + "--run", + metavar="TEST_NAME", + dest="selected_tests", + nargs="+", + default=[], + help="List of one or more external tests to run (separated by sapce).", + ) + running_mode.add_argument( + "--run-all", + dest="run_all", + default=False, + action="store_true", + help="Run all available external tests.", + ) + + return parser.parse_args() + + +def main(): + try: + args = parse_commandline() + args.cmd(vars(args)) + return os.EX_OK + except ExternalTestNotFound as exception: + print(f"Error: {exception}", file=sys.stderr) + return os.EX_NOINPUT + except RuntimeError as exception: + print(f"Error: {exception}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main())