Merge pull request #12214 from ethereum/parallelize-external-tests

Parallelize external tests

.circleci/config.yml

@@ -945,6 +945,8 @@ jobs:
         default: 14
     docker:
       - image: circleci/node:<<parameters.nodejs_version>>
+    # NOTE: Each external test does 3 separate compile&test runs
+    parallelism: 3
     environment:
       TERM: xterm
       COMPILE_ONLY: <<parameters.compile_only>>

.circleci/soltest_all.sh

@@ -28,27 +28,31 @@ set -e
 
 REPODIR="$(realpath "$(dirname "$0")"/..)"
 
+# shellcheck source=scripts/common.sh
+source "${REPODIR}/scripts/common.sh"
+
 EVM_VALUES=(homestead byzantium constantinople petersburg istanbul berlin london)
 DEFAULT_EVM=london
 [[ " ${EVM_VALUES[*]} " =~ $DEFAULT_EVM ]]
 OPTIMIZE_VALUES=(0 1)
 STEPS=$(( 1 + ${#EVM_VALUES[@]} * ${#OPTIMIZE_VALUES[@]} ))
 
-if (( CIRCLE_NODE_TOTAL )) && (( CIRCLE_NODE_TOTAL > 1 ))
+RUN_STEPS=$(circleci_select_steps "$(seq "$STEPS")")
-then
+printTask "Running steps $RUN_STEPS..."
-    RUN_STEPS=$(seq "$STEPS" | circleci tests split | xargs)
-else
-    RUN_STEPS=$(seq "$STEPS" | xargs)
-fi
-
-echo "Running steps $RUN_STEPS..."
 
 STEP=1
 
 
 # Run for ABI encoder v1, without SMTChecker tests.
-[[ " $RUN_STEPS " == *" $STEP "* ]] && EVM="${DEFAULT_EVM}" OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t !smtCheckerTests" "${REPODIR}/.circleci/soltest.sh"
+if circleci_step_selected "$RUN_STEPS" "$STEP"
-STEP=$((STEP + 1))
+then
+    EVM="${DEFAULT_EVM}" \
+    OPTIMIZE=1 \
+    ABI_ENCODER_V1=1 \
+    BOOST_TEST_ARGS="-t !smtCheckerTests" \
+    "${REPODIR}/.circleci/soltest.sh"
+fi
+((++STEP))
 
 for OPTIMIZE in "${OPTIMIZE_VALUES[@]}"
 do
@@ -63,13 +67,16 @@ do
         DISABLE_SMTCHECKER=""
         [ "${OPTIMIZE}" != "0" ] && DISABLE_SMTCHECKER="-t !smtCheckerTests"
 
-        [[ " $RUN_STEPS " == *" $STEP "* ]] && EVM="$EVM" OPTIMIZE="$OPTIMIZE" SOLTEST_FLAGS="$SOLTEST_FLAGS $ENFORCE_GAS_ARGS $EWASM_ARGS" BOOST_TEST_ARGS="-t !@nooptions $DISABLE_SMTCHECKER" "${REPODIR}/.circleci/soltest.sh"
+        if circleci_step_selected "$RUN_STEPS" "$STEP"
-        STEP=$((STEP + 1))
+        then
+            EVM="$EVM" \
+            OPTIMIZE="$OPTIMIZE" \
+            SOLTEST_FLAGS="$SOLTEST_FLAGS $ENFORCE_GAS_ARGS $EWASM_ARGS" \
+            BOOST_TEST_ARGS="-t !@nooptions $DISABLE_SMTCHECKER" \
+            "${REPODIR}/.circleci/soltest.sh"
+        fi
+        ((++STEP))
     done
 done
 
-if ((STEP != STEPS + 1))
+((STEP == STEPS + 1)) || assertFail "Step counter not properly adjusted!"
-then
-    echo "Step counter not properly adjusted!" >&2
-    exit 1
-fi

scripts/common.sh

@@ -208,3 +208,25 @@ function safe_kill
         kill -9 "$PID"
     fi
 }
+
+function circleci_select_steps
+{
+    local all_steps="$1"
+    (( $# == 1 )) || assertFail
+
+    if (( CIRCLE_NODE_TOTAL )) && (( CIRCLE_NODE_TOTAL > 1 ))
+    then
+        echo "$all_steps" | circleci tests split | xargs
+    else
+        echo "$all_steps" | xargs
+    fi
+}
+
+function circleci_step_selected
+{
+    local selected_steps="$1"
+    local step="$2"
+    [[ $step != *" "* ]] || assertFail "Step names must not contain spaces."
+
+    [[ " $selected_steps " == *" $step "* ]] || return 1
+}

test/externalTests/colony.sh

@@ -38,6 +38,10 @@ function colony_test
     local min_optimizer_level=3
     local max_optimizer_level=3
 
+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
     setup_solcjs "$DIR" "$SOLJSON"
     download_project "$repo" "$branch" "$DIR"
 
@@ -54,7 +58,7 @@ function colony_test
     replace_version_pragmas
     force_solc_modules "${DIR}/solc"
 
-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
         truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }

test/externalTests/common.sh

@@ -24,6 +24,15 @@ set -e
 
 CURRENT_EVM_VERSION=london
 
+function print_optimizer_levels_or_exit
+{
+    local selected_levels="$1"
+
+    [[ $selected_levels != "" ]] || { printWarning "No steps to run. Exiting."; exit 0; }
+
+    printLog "Selected optimizer levels: ${selected_levels}"
+}
+
 function verify_input
 {
     if [ ! -f "$1" ]; then

test/externalTests/ens.sh

@@ -38,6 +38,10 @@ function ens_test
     local min_optimizer_level=1
     local max_optimizer_level=3
 
+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
     setup_solcjs "$DIR" "$SOLJSON"
     download_project "$repo" "$branch" "$DIR"
 
@@ -52,7 +56,7 @@ function ens_test
     replace_version_pragmas
     force_solc_modules "${DIR}/solc"
 
-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
         truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }

test/externalTests/gnosis-v2.sh

@@ -39,6 +39,10 @@ function gnosis_safe_test
     local min_optimizer_level=2
     local max_optimizer_level=3
 
+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
     setup_solcjs "$DIR" "$SOLJSON"
     download_project "$repo" "$branch" "$DIR"
 
@@ -53,7 +57,7 @@ function gnosis_safe_test
     replace_version_pragmas
     force_solc_modules "${DIR}/solc"
 
-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
         truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }

test/externalTests/gnosis.sh

@@ -38,6 +38,10 @@ function gnosis_safe_test
     local min_optimizer_level=2
     local max_optimizer_level=3
 
+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
     setup_solcjs "$DIR" "$SOLJSON"
     download_project "$repo" "$branch" "$DIR"
 
@@ -51,7 +55,7 @@ function gnosis_safe_test
     replace_version_pragmas
     force_solc_modules "${DIR}/solc"
 
-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
         truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }

test/externalTests/zeppelin.sh

@@ -38,6 +38,10 @@ function zeppelin_test
     local min_optimizer_level=1
     local max_optimizer_level=3
 
+    local selected_optimizer_levels
+    selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
+    print_optimizer_levels_or_exit "$selected_optimizer_levels"
+
     setup_solcjs "$DIR" "$SOLJSON"
     download_project "$repo" "$branch" "$DIR"
 
@@ -48,7 +52,7 @@ function zeppelin_test
     replace_version_pragmas
     force_solc_modules "${DIR}/solc"
 
-    for level in $(seq "$min_optimizer_level" "$max_optimizer_level"); do
+    for level in $selected_optimizer_levels; do
         truffle_run_test "$config_file" "${DIR}/solc" "$level" compile_fn test_fn
     done
 }