diff --git a/.circleci/config.yml b/.circleci/config.yml index d5ea61ad2..65222f082 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,7 +202,7 @@ defaults: - run_cmdline_tests: &run_cmdline_tests name: command line tests no_output_timeout: 30m - command: ./test/cmdlineTests.sh + command: .circleci/parallel_cli_tests.py - run_docs_pragma_min_version: &run_docs_pragma_min_version name: docs pragma version check @@ -1053,6 +1053,7 @@ jobs: t_osx_cli: <<: *base_osx + parallelism: 7 # Should match number of tests in .circleci/cli.sh steps: - checkout - when: @@ -1147,6 +1148,7 @@ jobs: t_ubu_cli: &t_ubu_cli <<: *base_ubuntu2204_small + parallelism: 7 # Should match number of tests in .circleci/cli.sh <<: *steps_cmdline_tests t_ubu_force_release_cli: &t_ubu_force_release_cli @@ -1164,6 +1166,7 @@ jobs: t_ubu_asan_cli: # Runs slightly faster on medium but we only run it nightly so efficiency matters more. <<: *base_ubuntu2204 + parallelism: 7 # Should match number of tests in .circleci/cli.sh environment: TERM: xterm ASAN_OPTIONS: check_initialization_order=true:detect_stack_use_after_return=true:strict_init_order=true:strict_string_checks=true:detect_invalid_pointer_pairs=2 @@ -1205,6 +1208,7 @@ jobs: t_ubu_ubsan_clang_cli: <<: *base_ubuntu2204_clang + parallelism: 7 # Should match number of tests in .circleci/cli.sh <<: *steps_cmdline_tests t_ems_solcjs: diff --git a/.circleci/parallel_cli_tests.py b/.circleci/parallel_cli_tests.py new file mode 100755 index 000000000..ef7892180 --- /dev/null +++ b/.circleci/parallel_cli_tests.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import subprocess +import sys + +# Slowest CLI tests, whose execution takes time on the order of minutes (as of June 2023). +# When adding/removing items here, remember to update `parallelism` value in jobs that run this script. +# TODO: We should switch to time-based splitting but that requires JUnit XML report support in cmdlineTests.sh. +tests_to_run_in_parallel = [ + '~ast_import_export', # ~7 min + '~ast_export_with_stop_after_parsing', # ~4 min + '~soljson_via_fuzzer', # ~3 min + '~via_ir_equivalence', # ~1 min + '~compilation_tests', # ~1 min + '~documentation_examples', # ~1 min + '*', # This item represents all the remaining tests +] + +# Ask CircleCI to select a subset of tests for this parallel execution. +# If `parallelism` in CI config is set correctly, we should get just one but we can handle any split. +selected_tests = subprocess.check_output( + ['circleci', 'tests', 'split'], + input='\n'.join(tests_to_run_in_parallel), + encoding='ascii', +).strip().split('\n') +selected_tests = set(selected_tests) - {''} +excluded_tests = set(tests_to_run_in_parallel) - selected_tests +assert selected_tests.issubset(set(tests_to_run_in_parallel)) + +if len(selected_tests) == 0: + print("No tests to run.") + sys.exit(0) + +if '*' in selected_tests: + filters = [arg for test_name in excluded_tests for arg in ['--exclude', test_name]] +else: + filters = list(selected_tests) + +subprocess.run( + ['test/cmdlineTests.sh'] + filters, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + check=True, +)