diff --git a/.circleci/config.yml b/.circleci/config.yml index 8123fefe8..f592a8703 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1054,8 +1054,9 @@ jobs: docker: - image: circleci/node:<> resource_class: <> - # NOTE: Each external test does 3 separate compile&test runs - parallelism: 3 + # NOTE: Each external test runs up to 6 independent settings presets. If parallelism is higher than + # actual preset count, some runs will exit immediately. If it's lower, some runs will get more than one preset. + parallelism: 6 environment: TERM: xterm COMPILE_ONLY: <> diff --git a/scripts/common.sh b/scripts/common.sh index 3e67c4f56..4a98c0072 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -211,6 +211,7 @@ function safe_kill function circleci_select_steps { + # We expect multiple lines in $all_steps, one step per line local all_steps="$1" (( $# == 1 )) || assertFail @@ -222,11 +223,26 @@ function circleci_select_steps fi } +function circleci_select_steps_multiarg +{ + # We expect multiple arguments, one step per argument. + circleci_select_steps "$(printf '%s\n' "$@")" +} + function circleci_step_selected { local selected_steps="$1" local step="$2" + (( $# == 2 )) || assertFail [[ $step != *" "* ]] || assertFail "Step names must not contain spaces." [[ " $selected_steps " == *" $step "* ]] || return 1 } + +function first_word +{ + local words="$1" + (( $# == 1 )) || assertFail + + echo "$words" | cut -d " " -f 1 +} diff --git a/test/externalTests/colony.sh b/test/externalTests/colony.sh index 65a2183bc..d5b57f024 100755 --- a/test/externalTests/colony.sh +++ b/test/externalTests/colony.sh @@ -36,20 +36,25 @@ function colony_test local repo="https://github.com/solidity-external-tests/colonyNetwork.git" local branch=develop_080 local config_file="truffle.js" - # On levels 1 and 2 it compiles but tests run out of gas - local min_optimizer_level=3 - local max_optimizer_level=3 + local settings_presets=( + #ir-no-optimize # Compiles but tests run out of gas + #ir-optimize-evm-only # Compiles but tests run out of gas + ir-optimize-evm+yul + #legacy-no-optimize # Compiles but tests run out of gas + #legacy-optimize-evm-only # Compiles but tests run out of gas + legacy-optimize-evm+yul + ) - local selected_optimizer_levels - selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")") - print_optimizer_levels_or_exit "$selected_optimizer_levels" + local selected_optimizer_presets + selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}") + print_optimizer_presets_or_exit "$selected_optimizer_presets" setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" download_project "$repo" "$branch" "$DIR" [[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH" neutralize_package_json_hooks - force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level" + force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")" yarn install git submodule update --init @@ -61,8 +66,8 @@ function colony_test replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc" - for level in $selected_optimizer_levels; do - truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn + for preset in $selected_optimizer_presets; do + truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn done } diff --git a/test/externalTests/common.sh b/test/externalTests/common.sh index 80a9dec32..012a21b2f 100644 --- a/test/externalTests/common.sh +++ b/test/externalTests/common.sh @@ -24,13 +24,13 @@ set -e CURRENT_EVM_VERSION=london -function print_optimizer_levels_or_exit +function print_optimizer_presets_or_exit { - local selected_levels="$1" + local selected_presets="$1" - [[ $selected_levels != "" ]] || { printWarning "No steps to run. Exiting."; exit 0; } + [[ $selected_presets != "" ]] || { printWarning "No presets to run. Exiting."; exit 0; } - printLog "Selected optimizer levels: ${selected_levels}" + printLog "Selected settings presets: ${selected_presets}" } function verify_input @@ -141,7 +141,7 @@ function force_truffle_compiler_settings local config_file="$1" local binary_type="$2" local solc_path="$3" - local level="$4" + local preset="$4" local evm_version="${5:-"$CURRENT_EVM_VERSION"}" [[ $binary_type == native || $binary_type == solcjs ]] || assertFail @@ -153,14 +153,16 @@ function force_truffle_compiler_settings echo "Config file: $config_file" echo "Binary type: $binary_type" echo "Compiler path: $solc_path" - echo "Optimization level: $level" - echo "Optimizer settings: $(optimizer_settings_for_level "$level")" + echo "Settings preset: ${preset}" + echo "Settings: $(settings_from_preset "$preset" "$evm_version")" echo "EVM version: $evm_version" + echo "Compiler version: ${SOLCVERSION_SHORT}" + echo "Compiler version (full): ${SOLCVERSION}" echo "-------------------------------------" # Forcing the settings should always work by just overwriting the solc object. Forcing them by using a # dedicated settings objects should only be the fallback. - echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$level" "$evm_version");" >> "$config_file" + echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$preset" "$evm_version");" >> "$config_file" } function force_hardhat_compiler_binary @@ -180,14 +182,14 @@ function force_hardhat_compiler_binary function force_hardhat_compiler_settings { local config_file="$1" - local level="$2" + local preset="$2" local evm_version="${3:-"$CURRENT_EVM_VERSION"}" printLog "Configuring Hardhat..." echo "-------------------------------------" echo "Config file: ${config_file}" - echo "Optimization level: ${level}" - echo "Optimizer settings: $(optimizer_settings_for_level "$level")" + echo "Settings preset: ${preset}" + echo "Settings: $(settings_from_preset "$preset" "$evm_version")" echo "EVM version: ${evm_version}" echo "Compiler version: ${SOLCVERSION_SHORT}" echo "Compiler version (full): ${SOLCVERSION}" @@ -195,7 +197,7 @@ function force_hardhat_compiler_settings { echo -n 'module.exports["solidity"] = ' - hardhat_compiler_settings "$SOLCVERSION_SHORT" "$level" "$evm_version" + hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version" } >> "$config_file" } @@ -236,22 +238,27 @@ function run_test replace_version_pragmas printLog "Running compile function..." - $compile_fn + time $compile_fn printLog "Running test function..." $test_fn } -function optimizer_settings_for_level +function settings_from_preset { - local level="$1" + local preset="$1" + local evm_version="$2" - case "$level" in - 1) echo "{enabled: false}" ;; - 2) echo "{enabled: true, details: {yul: false}}" ;; - 3) echo "{enabled: true, details: {yul: true}}" ;; + case "$preset" in + # NOTE: Remember to update `parallelism` of `t_ems_ext` job in CI config if you add/remove presets + legacy-no-optimize) echo "{evmVersion: '${evm_version}', viaIR: false, optimizer: {enabled: false}}" ;; + ir-no-optimize) echo "{evmVersion: '${evm_version}', viaIR: true, optimizer: {enabled: false}}" ;; + legacy-optimize-evm-only) echo "{evmVersion: '${evm_version}', viaIR: false, optimizer: {enabled: true, details: {yul: false}}}" ;; + ir-optimize-evm-only) echo "{evmVersion: '${evm_version}', viaIR: true, optimizer: {enabled: true, details: {yul: false}}}" ;; + legacy-optimize-evm+yul) echo "{evmVersion: '${evm_version}', viaIR: false, optimizer: {enabled: true, details: {yul: true}}}" ;; + ir-optimize-evm+yul) echo "{evmVersion: '${evm_version}', viaIR: true, optimizer: {enabled: true, details: {yul: true}}}" ;; *) - fail "Optimizer level not found. Please define OPTIMIZER_LEVEL=[1, 2, 3]" + fail "Unknown settings preset: '${preset}'." ;; esac } @@ -269,16 +276,13 @@ function replace_global_solc function truffle_compiler_settings { local solc_path="$1" - local level="$2" + local preset="$2" local evm_version="$3" echo "{" echo " solc: {" echo " version: \"${solc_path}\"," - echo " settings: {" - echo " optimizer: $(optimizer_settings_for_level "$level")," - echo " evmVersion: \"${evm_version}\"" - echo " }" + echo " settings: $(settings_from_preset "$preset" "$evm_version")" echo " }" echo "}" } @@ -310,15 +314,12 @@ function hardhat_solc_build_subtask { function hardhat_compiler_settings { local solc_version="$1" - local level="$2" + local preset="$2" local evm_version="$3" echo "{" echo " version: '${solc_version}'," - echo " settings: {" - echo " optimizer: $(optimizer_settings_for_level "$level")," - echo " evmVersion: '${evm_version}'" - echo " }" + echo " settings: $(settings_from_preset "$preset" "$evm_version")" echo "}" } @@ -329,7 +330,7 @@ function compile_and_run_test local verify_fn="$3" printLog "Running compile function..." - $compile_fn + time $compile_fn $verify_fn "$SOLCVERSION_SHORT" "$SOLCVERSION" if [[ "$COMPILE_ONLY" == 1 ]]; then @@ -345,24 +346,24 @@ function truffle_run_test local config_file="$1" local binary_type="$2" local solc_path="$3" - local optimizer_level="$4" + local preset="$4" local compile_fn="$5" local test_fn="$6" truffle_clean - force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$optimizer_level" + force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$preset" compile_and_run_test compile_fn test_fn truffle_verify_compiler_version } function hardhat_run_test { local config_file="$1" - local optimizer_level="$2" + local preset="$2" local compile_fn="$3" local test_fn="$4" hardhat_clean - force_hardhat_compiler_settings "$config_file" "$optimizer_level" + force_hardhat_compiler_settings "$config_file" "$preset" compile_and_run_test compile_fn test_fn hardhat_verify_compiler_version } diff --git a/test/externalTests/ens.sh b/test/externalTests/ens.sh index 7b354acde..c0bc4b558 100755 --- a/test/externalTests/ens.sh +++ b/test/externalTests/ens.sh @@ -36,12 +36,18 @@ function ens_test local repo="https://github.com/ensdomains/ens.git" local branch=master local config_file="truffle.js" - local min_optimizer_level=1 - local max_optimizer_level=3 + local settings_presets=( + #ir-no-optimize # "YulException: Variable var_ttl_236 is 1 slot(s) too deep inside the stack." + #ir-optimize-evm-only # "YulException: Variable var_ttl_236 is 1 slot(s) too deep inside the stack." + ir-optimize-evm+yul + legacy-no-optimize + legacy-optimize-evm-only + legacy-optimize-evm+yul + ) - local selected_optimizer_levels - selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")") - print_optimizer_levels_or_exit "$selected_optimizer_levels" + local selected_optimizer_presets + selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}") + print_optimizer_presets_or_exit "$selected_optimizer_presets" setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" download_project "$repo" "$branch" "$DIR" @@ -52,14 +58,14 @@ function ens_test neutralize_package_lock neutralize_package_json_hooks - force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level" + force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")" npm install replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc" - for level in $selected_optimizer_levels; do - truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn + for preset in $selected_optimizer_presets; do + truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn done } diff --git a/test/externalTests/gnosis-v2.sh b/test/externalTests/gnosis-v2.sh index b71749a06..bae0f0f1e 100755 --- a/test/externalTests/gnosis-v2.sh +++ b/test/externalTests/gnosis-v2.sh @@ -36,13 +36,18 @@ function gnosis_safe_test local repo="https://github.com/solidity-external-tests/safe-contracts.git" local branch=v2_080 local config_file="truffle-config.js" - # level 1: "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit" - local min_optimizer_level=2 - local max_optimizer_level=3 + local settings_presets=( + #ir-no-optimize # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + #ir-optimize-evm-only # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + ir-optimize-evm+yul + #legacy-no-optimize # "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit" + legacy-optimize-evm-only + legacy-optimize-evm+yul + ) - local selected_optimizer_levels - selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")") - print_optimizer_levels_or_exit "$selected_optimizer_levels" + local selected_optimizer_presets + selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}") + print_optimizer_presets_or_exit "$selected_optimizer_presets" setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" download_project "$repo" "$branch" "$DIR" @@ -53,14 +58,14 @@ function gnosis_safe_test neutralize_package_lock neutralize_package_json_hooks - force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level" + force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")" npm install --package-lock replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc" - for level in $selected_optimizer_levels; do - truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn + for preset in $selected_optimizer_presets; do + truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn done } diff --git a/test/externalTests/gnosis.sh b/test/externalTests/gnosis.sh index 97208660b..562ab156f 100755 --- a/test/externalTests/gnosis.sh +++ b/test/externalTests/gnosis.sh @@ -36,13 +36,18 @@ function gnosis_safe_test local repo="https://github.com/solidity-external-tests/safe-contracts.git" local branch=development_080 local config_file="truffle-config.js" - # levels 1 and 2: "Stack too deep" error - local min_optimizer_level=3 - local max_optimizer_level=3 + local settings_presets=( + #ir-no-optimize # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + #ir-optimize-evm-only # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + ir-optimize-evm+yul + #legacy-no-optimize # "Stack too deep" error + #legacy-optimize-evm-only # "Stack too deep" error + legacy-optimize-evm+yul + ) - local selected_optimizer_levels - selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")") - print_optimizer_levels_or_exit "$selected_optimizer_levels" + local selected_optimizer_presets + selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}") + print_optimizer_presets_or_exit "$selected_optimizer_presets" setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" download_project "$repo" "$branch" "$DIR" @@ -52,14 +57,14 @@ function gnosis_safe_test neutralize_package_lock neutralize_package_json_hooks - force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level" + force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")" npm install --package-lock replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc" - for level in $selected_optimizer_levels; do - truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn + for preset in $selected_optimizer_presets; do + truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn done } diff --git a/test/externalTests/zeppelin.sh b/test/externalTests/zeppelin.sh index 0759a3aec..89f3d6b42 100755 --- a/test/externalTests/zeppelin.sh +++ b/test/externalTests/zeppelin.sh @@ -36,25 +36,31 @@ function zeppelin_test local repo="https://github.com/OpenZeppelin/openzeppelin-contracts.git" local branch=master local config_file="hardhat.config.js" - local min_optimizer_level=1 - local max_optimizer_level=3 + local settings_presets=( + #ir-no-optimize # "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." + #ir-optimize-evm-only # "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." + #ir-optimize-evm+yul # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115 + legacy-no-optimize + legacy-optimize-evm-only + legacy-optimize-evm+yul + ) - local selected_optimizer_levels - selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")") - print_optimizer_levels_or_exit "$selected_optimizer_levels" + local selected_optimizer_presets + selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}") + print_optimizer_presets_or_exit "$selected_optimizer_presets" setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" download_project "$repo" "$branch" "$DIR" neutralize_package_json_hooks force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" - force_hardhat_compiler_settings "$config_file" "$min_optimizer_level" + force_hardhat_compiler_settings "$config_file" "$(first_word "$selected_optimizer_presets")" npm install replace_version_pragmas - for level in $selected_optimizer_levels; do - hardhat_run_test "$config_file" "$level" compile_fn test_fn + for preset in $selected_optimizer_presets; do + hardhat_run_test "$config_file" "$preset" compile_fn test_fn done }