Merge pull request #14339 from ethereum/parallelize-cli-tests

Parallelize CLI tests

.circleci/config.yml

@@ -202,7 +202,7 @@ defaults:
   - run_cmdline_tests: &run_cmdline_tests
       name: command line tests
       no_output_timeout: 30m
-      command: ./test/cmdlineTests.sh
+      command: .circleci/parallel_cli_tests.py
 
   - run_docs_pragma_min_version: &run_docs_pragma_min_version
       name: docs pragma version check
@@ -1053,6 +1053,7 @@ jobs:
 
   t_osx_cli:
     <<: *base_osx
+    parallelism: 7 # Should match number of tests in .circleci/cli.sh
     steps:
       - checkout
       - when:
@@ -1147,6 +1148,7 @@ jobs:
 
   t_ubu_cli: &t_ubu_cli
     <<: *base_ubuntu2204_small
+    parallelism: 7 # Should match number of tests in .circleci/cli.sh
     <<: *steps_cmdline_tests
 
   t_ubu_force_release_cli: &t_ubu_force_release_cli
@@ -1164,6 +1166,7 @@ jobs:
   t_ubu_asan_cli:
     # Runs slightly faster on medium but we only run it nightly so efficiency matters more.
     <<: *base_ubuntu2204
+    parallelism: 7 # Should match number of tests in .circleci/cli.sh
     environment:
       TERM: xterm
       ASAN_OPTIONS: check_initialization_order=true:detect_stack_use_after_return=true:strict_init_order=true:strict_string_checks=true:detect_invalid_pointer_pairs=2
@@ -1205,6 +1208,7 @@ jobs:
 
   t_ubu_ubsan_clang_cli:
     <<: *base_ubuntu2204_clang
+    parallelism: 7 # Should match number of tests in .circleci/cli.sh
     <<: *steps_cmdline_tests
 
   t_ems_solcjs:

.circleci/parallel_cli_tests.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+
+# Slowest CLI tests, whose execution takes time on the order of minutes (as of June 2023).
+# When adding/removing items here, remember to update `parallelism` value in jobs that run this script.
+# TODO: We should switch to time-based splitting but that requires JUnit XML report support in cmdlineTests.sh.
+tests_to_run_in_parallel = [
+    '~ast_import_export',                   # ~7 min
+    '~ast_export_with_stop_after_parsing',  # ~4 min
+    '~soljson_via_fuzzer',                  # ~3 min
+    '~via_ir_equivalence',                  # ~1 min
+    '~compilation_tests',                   # ~1 min
+    '~documentation_examples',              # ~1 min
+    '*',                                    # This item represents all the remaining tests
+]
+
+# Ask CircleCI to select a subset of tests for this parallel execution.
+# If `parallelism` in CI config is set correctly, we should get just one but we can handle any split.
+selected_tests = subprocess.check_output(
+    ['circleci', 'tests', 'split'],
+    input='\n'.join(tests_to_run_in_parallel),
+    encoding='ascii',
+).strip().split('\n')
+selected_tests = set(selected_tests) - {''}
+excluded_tests = set(tests_to_run_in_parallel) - selected_tests
+assert selected_tests.issubset(set(tests_to_run_in_parallel))
+
+if len(selected_tests) == 0:
+    print("No tests to run.")
+    sys.exit(0)
+
+if '*' in selected_tests:
+    filters = [arg for test_name in excluded_tests for arg in ['--exclude', test_name]]
+else:
+    filters = list(selected_tests)
+
+subprocess.run(
+    ['test/cmdlineTests.sh'] + filters,
+    stdin=sys.stdin,
+    stdout=sys.stdout,
+    stderr=sys.stderr,
+    check=True,
+)