Merge pull request #12214 from ethereum/parallelize-external-tests

Parallelize external tests
2023-10-03 13:03:40 +00:00 · 2021-11-18 17:27:57 +01:00 · 2021-11-18 17:27:57 +01:00 · 2aeeef83f2
commit 2aeeef83f2
parent cbb3e07480 834f9d3a6e
9 changed files with 82 additions and 22 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -945,6 +945,8 @@ jobs:
        default: 14
    docker:
      - image: circleci/node:<<parameters.nodejs_version>>
+    # NOTE: Each external test does 3 separate compile&test runs
+    parallelism: 3
    environment:
      TERM: xterm
      COMPILE_ONLY: <<parameters.compile_only>>
--- a/.circleci/soltest_all.sh
+++ b/.circleci/soltest_all.sh
@ -28,27 +28,31 @@ set -e

 REPODIR="$(realpath "$(dirname "$0")"/..)"

+# shellcheck source=scripts/common.sh
+source "${REPODIR}/scripts/common.sh"
+
 EVM_VALUES=(homestead byzantium constantinople petersburg istanbul berlin london)
 DEFAULT_EVM=london
 [[ " ${EVM_VALUES[*]} " =~ $DEFAULT_EVM ]]
 OPTIMIZE_VALUES=(0 1)
 STEPS=$(( 1 + ${#EVM_VALUES[@]} * ${#OPTIMIZE_VALUES[@]} ))

-if (( CIRCLE_NODE_TOTAL )) && (( CIRCLE_NODE_TOTAL > 1 ))
-then
-    RUN_STEPS=$(seq "$STEPS" | circleci tests split | xargs)
-else
-    RUN_STEPS=$(seq "$STEPS" | xargs)
-fi
-
-echo "Running steps $RUN_STEPS..."
+RUN_STEPS=$(circleci_select_steps "$(seq "$STEPS")")
+printTask "Running steps $RUN_STEPS..."

 STEP=1


 # Run for ABI encoder v1, without SMTChecker tests.
-[[ " $RUN_STEPS " == *" $STEP "* ]] && EVM="${DEFAULT_EVM}" OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t !smtCheckerTests" "${REPODIR}/.circleci/soltest.sh"
-STEP=$((STEP + 1))
+if circleci_step_selected "$RUN_STEPS" "$STEP"
+then
+    EVM="${DEFAULT_EVM}" \
+    OPTIMIZE=1 \
+    ABI_ENCODER_V1=1 \
+    BOOST_TEST_ARGS="-t !smtCheckerTests" \
+    "${REPODIR}/.circleci/soltest.sh"
+fi
+((++STEP))

 for OPTIMIZE in "${OPTIMIZE_VALUES[@]}"
 do
@ -63,13 +67,16 @@ do
        DISABLE_SMTCHECKER=""
        [ "${OPTIMIZE}" != "0" ] && DISABLE_SMTCHECKER="-t !smtCheckerTests"

-        [[ " $RUN_STEPS " == *" $STEP "* ]] && EVM="$EVM" OPTIMIZE="$OPTIMIZE" SOLTEST_FLAGS="$SOLTEST_FLAGS $ENFORCE_GAS_ARGS $EWASM_ARGS" BOOST_TEST_ARGS="-t !@nooptions $DISABLE_SMTCHECKER" "${REPODIR}/.circleci/soltest.sh"
-        STEP=$((STEP + 1))
+        if circleci_step_selected "$RUN_STEPS" "$STEP"
+        then
+            EVM="$EVM" \
+            OPTIMIZE="$OPTIMIZE" \
+            SOLTEST_FLAGS="$SOLTEST_FLAGS $ENFORCE_GAS_ARGS $EWASM_ARGS" \
+            BOOST_TEST_ARGS="-t !@nooptions $DISABLE_SMTCHECKER" \
+            "${REPODIR}/.circleci/soltest.sh"
+        fi
+        ((++STEP))
    done
 done

-if ((STEP != STEPS + 1))
-then
-    echo "Step counter not properly adjusted!" >&2
-    exit 1
-fi
+((STEP == STEPS + 1)) || assertFail "Step counter not properly adjusted!"
--- a/scripts/common.sh
+++ b/scripts/common.sh
@ -208,3 +208,25 @@ function safe_kill
        kill -9 "$PID"
    fi
 }
+
+function circleci_select_steps
+{
+    local all_steps="$1"
+    (( $# == 1 )) || assertFail
+
+    if (( CIRCLE_NODE_TOTAL )) && (( CIRCLE_NODE_TOTAL > 1 ))
+    then
+        echo "$all_steps" | circleci tests split | xargs
+    else
+        echo "$all_steps" | xargs
+    fi
+}
+
+function circleci_step_selected
+{
+    local selected_steps="$1"
+    local step="$2"
+    [[ $step != *" "* ]] || assertFail "Step names must not contain spaces."
+
+    [[ " $selected_steps " == *" $step "* ]] || return 1
+}
--- a/test/externalTests/colony.sh
+++ b/test/externalTests/colony.sh
@ -38,6 +38,10 @@ function colony_test
    local min_optimizer_level=3
    local max_optimizer_level=3

+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
    setup_solcjs "$DIR" "$SOLJSON"
    download_project "$repo" "$branch" "$DIR"

@ -54,7 +58,7 @@ function colony_test
    replace_version_pragmas
    force_solc_modules "${DIR}/solc"

-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }
--- a/test/externalTests/common.sh
+++ b/test/externalTests/common.sh
@ -24,6 +24,15 @@ set -e

 CURRENT_EVM_VERSION=london

+function print_optimizer_levels_or_exit
+{
+    local selected_levels="$1"
+
+    [[ $selected_levels != "" ]] || { printWarning "No steps to run. Exiting."; exit 0; }
+
+    printLog "Selected optimizer levels: ${selected_levels}"
+}
+
 function verify_input
 {
    if [ ! -f "$1" ]; then
--- a/test/externalTests/ens.sh
+++ b/test/externalTests/ens.sh
@ -38,6 +38,10 @@ function ens_test
    local min_optimizer_level=1
    local max_optimizer_level=3

+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
    setup_solcjs "$DIR" "$SOLJSON"
    download_project "$repo" "$branch" "$DIR"

@ -52,7 +56,7 @@ function ens_test
    replace_version_pragmas
    force_solc_modules "${DIR}/solc"

-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }
--- a/test/externalTests/gnosis-v2.sh
+++ b/test/externalTests/gnosis-v2.sh
@ -39,6 +39,10 @@ function gnosis_safe_test
    local min_optimizer_level=2
    local max_optimizer_level=3

+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
    setup_solcjs "$DIR" "$SOLJSON"
    download_project "$repo" "$branch" "$DIR"

@ -53,7 +57,7 @@ function gnosis_safe_test
    replace_version_pragmas
    force_solc_modules "${DIR}/solc"

-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }
--- a/test/externalTests/gnosis.sh
+++ b/test/externalTests/gnosis.sh
@ -38,6 +38,10 @@ function gnosis_safe_test
    local min_optimizer_level=2
    local max_optimizer_level=3

+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
    setup_solcjs "$DIR" "$SOLJSON"
    download_project "$repo" "$branch" "$DIR"

@ -51,7 +55,7 @@ function gnosis_safe_test
    replace_version_pragmas
    force_solc_modules "${DIR}/solc"

-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }
--- a/test/externalTests/zeppelin.sh
+++ b/test/externalTests/zeppelin.sh
@ -38,6 +38,10 @@ function zeppelin_test
    local min_optimizer_level=1
    local max_optimizer_level=3

+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
    setup_solcjs "$DIR" "$SOLJSON"
    download_project "$repo" "$branch" "$DIR"

@ -48,7 +52,7 @@ function zeppelin_test
    replace_version_pragmas
    force_solc_modules "${DIR}/solc"

-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
        truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
    done
 }