Merge pull request #14064 from ethereum/external-tests-refactor

External tests script refactor

test/externalTests.sh

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-#------------------------------------------------------------------------------
-# Bash script to run external Solidity tests.
-#
-# Argument: Path to soljson.js to test.
-#
-# Requires npm, networking access and git to download the tests.
-#
-# ------------------------------------------------------------------------------
-# This file is part of solidity.
-#
-# solidity is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# solidity is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with solidity.  If not, see <http://www.gnu.org/licenses/>
-#
-# (c) 2016 solidity contributors.
-#------------------------------------------------------------------------------
-
-set -e
-
-source scripts/common.sh
-source test/externalTests/common.sh
-
-REPO_ROOT=$(realpath "$(dirname "$0")/..")
-
-verify_input "$@"
-
-printTask "Running external tests..."
-
-"${REPO_ROOT}/test/externalTests/zeppelin.sh" "$@"
-"${REPO_ROOT}/test/externalTests/gnosis.sh" "$@"
-"${REPO_ROOT}/test/externalTests/colony.sh" "$@"
-"${REPO_ROOT}/test/externalTests/ens.sh" "$@"
-"${REPO_ROOT}/test/externalTests/trident.sh" "$@"
-"${REPO_ROOT}/test/externalTests/euler.sh" "$@"
-"${REPO_ROOT}/test/externalTests/yield-liquidator.sh" "$@"
-"${REPO_ROOT}/test/externalTests/bleeps.sh" "$@"
-"${REPO_ROOT}/test/externalTests/pool-together.sh" "$@"
-"${REPO_ROOT}/test/externalTests/perpetual-pools.sh" "$@"
-"${REPO_ROOT}/test/externalTests/uniswap.sh" "$@"
-"${REPO_ROOT}/test/externalTests/prb-math.sh" "$@"
-"${REPO_ROOT}/test/externalTests/elementfi.sh" "$@"
-"${REPO_ROOT}/test/externalTests/brink.sh" "$@"
-"${REPO_ROOT}/test/externalTests/chainlink.sh" "$@"
-"${REPO_ROOT}/test/externalTests/gp2.sh" "$@"

test/externalTests/bleeps.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/brink.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/chainlink.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/colony.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/elementfi.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/ens.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/euler.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/gnosis.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/gp2.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/perpetual-pools.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/pool-together.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/prb-math.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/solc-js/solc-js.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 SOLJSON="$1"
 VERSION="$2"

test/externalTests/trident.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/uniswap.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/yield-liquidator.sh

@@ -22,7 +22,7 @@
 set -e
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/externalTests/zeppelin.sh

@@ -27,7 +27,7 @@ set -e
 export NODE_OPTIONS="--max-old-space-size=4096"
 
 source scripts/common.sh
-source test/externalTests/common.sh
+source scripts/externalTests/common.sh
 
 REPO_ROOT=$(realpath "$(dirname "$0")/../..")
 

test/external_tests.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2023 solidity contributors.
+# ------------------------------------------------------------------------------
+
+from argparse import ArgumentParser, Namespace
+import os
+from pathlib import Path
+import sys
+import subprocess
+
+EXTERNAL_TESTS_DIR = Path(__file__).parent / "externalTests"
+
+
+class ExternalTestNotFound(Exception):
+    pass
+
+
+def detect_external_tests() -> dict:
+    return {
+        file_path.stem: file_path
+        for file_path in Path(EXTERNAL_TESTS_DIR).iterdir()
+        if file_path.is_file() and file_path.suffix == ".sh"
+    }
+
+
+def display_available_external_tests(_):
+    print("Available external tests:")
+    print(*detect_external_tests().keys())
+
+
+def run_test_scripts(solc_binary_type: str, solc_binary_path: Path, tests: dict):
+    for test_name, test_script_path in tests.items():
+        print(f"Running {test_name} external test...")
+        subprocess.run(
+            [test_script_path, solc_binary_type, solc_binary_path],
+            check=True
+        )
+
+
+def run_external_tests(args: dict):
+    solc_binary_type = args["solc_binary_type"]
+    solc_binary_path = args["solc_binary_path"]
+
+    all_test_scripts = detect_external_tests()
+    if args["run_all"]:
+        run_test_scripts(solc_binary_type, solc_binary_path, all_test_scripts)
+    else:
+        selected_tests = args["selected_tests"]
+        if selected_tests:
+            unrecognized_tests = set(selected_tests) - set(all_test_scripts.keys())
+            if unrecognized_tests != set():
+                raise ExternalTestNotFound(
+                    f"External test(s) not found: {', '.join(unrecognized_tests)}"
+                )
+            run_test_scripts(
+                solc_binary_type,
+                solc_binary_path,
+                {k: all_test_scripts[k] for k in selected_tests},
+            )
+    raise ExternalTestNotFound(
+        "External test was not selected. Please use --run or --run-all option"
+    )
+
+
+def parse_commandline() -> Namespace:
+    script_description = "Script to run external Solidity tests."
+
+    parser = ArgumentParser(description=script_description)
+    subparser = parser.add_subparsers()
+    list_command = subparser.add_parser(
+        "list",
+        help="List all available external tests.",
+    )
+    list_command.set_defaults(cmd=display_available_external_tests)
+
+    run_command = subparser.add_parser(
+        "test",
+        help="Run external tests.",
+    )
+    run_command.set_defaults(cmd=run_external_tests)
+
+    run_command.add_argument(
+        "--solc-binary-type",
+        dest="solc_binary_type",
+        type=str,
+        required=True,
+        choices=["native", "solcjs"],
+        help="Type of the solidity compiler binary to be used.",
+    )
+    run_command.add_argument(
+        "--solc-binary-path",
+        dest="solc_binary_path",
+        type=Path,
+        required=True,
+        help="Path to the solidity compiler binary.",
+    )
+
+    running_mode = run_command.add_mutually_exclusive_group()
+    running_mode.add_argument(
+        "--run",
+        metavar="TEST_NAME",
+        dest="selected_tests",
+        nargs="+",
+        default=[],
+        help="List of one or more external tests to run (separated by sapce).",
+    )
+    running_mode.add_argument(
+        "--run-all",
+        dest="run_all",
+        default=False,
+        action="store_true",
+        help="Run all available external tests.",
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    try:
+        args = parse_commandline()
+        args.cmd(vars(args))
+        return os.EX_OK
+    except ExternalTestNotFound as exception:
+        print(f"Error: {exception}", file=sys.stderr)
+        return os.EX_NOINPUT
+    except RuntimeError as exception:
+        print(f"Error: {exception}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())