From e9f3f9361ddf1526be166e1f8ef2a5a535beec07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 1 Mar 2022 17:48:46 +0100 Subject: [PATCH] Re-enable running tests in external tests via IR in cases where they don't pass due to Hardhat heuristics --- test/externalTests/elementfi.sh | 12 +++++++++--- test/externalTests/gnosis.sh | 10 +++++++--- test/externalTests/prb-math.sh | 25 ++++++++++++++++++++++--- test/externalTests/zeppelin.sh | 18 +++++++++++++++--- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/test/externalTests/elementfi.sh b/test/externalTests/elementfi.sh index d8d3fd6cb..74ed93a5c 100755 --- a/test/externalTests/elementfi.sh +++ b/test/externalTests/elementfi.sh @@ -42,13 +42,12 @@ function elementfi_test local config_file="hardhat.config.ts" local config_var=config - local compile_only_presets=( - ir-optimize-evm+yul # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115 - ) + local compile_only_presets=() local settings_presets=( "${compile_only_presets[@]}" #ir-no-optimize # Compilation fails with "YulException: Variable var_amount_9311 is 10 slot(s) too deep inside the stack." #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_amount_9311 is 10 slot(s) too deep inside the stack." + ir-optimize-evm+yul legacy-no-optimize legacy-optimize-evm-only legacy-optimize-evm+yul @@ -89,6 +88,13 @@ function elementfi_test # TODO: Remove when https://github.com/element-fi/elf-contracts/issues/243 is fixed. sed -i 's|^\s*require(_expiration - block\.timestamp < _unitSeconds);\s*$||g' contracts/ConvergentCurvePool.sol + # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables + # them for other presets but that's fine - we want same code run for benchmarks to be comparable. + # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115). + sed -i 's|it(\("fails to withdraw more shares than in balance"\)|it.skip(\1|g' test/compoundAssetProxyTest.ts + sed -i 's|it(\("should prevent withdrawal of Principal Tokens and Interest Tokens before the tranche expires "\)|it.skip(\1|g' test/trancheTest.ts + sed -i 's|it(\("should prevent withdrawal of more Principal Tokens and Interest Tokens than the user has"\)|it.skip(\1|g' test/trancheTest.ts + # This test file is very flaky. There's one particular cases that fails randomly (see # https://github.com/element-fi/elf-contracts/issues/240) but some others also depends on an external # service which makes tests time out when that service is down. diff --git a/test/externalTests/gnosis.sh b/test/externalTests/gnosis.sh index ec09257fb..ccfc6cf5c 100755 --- a/test/externalTests/gnosis.sh +++ b/test/externalTests/gnosis.sh @@ -42,13 +42,12 @@ function gnosis_safe_test local config_file="hardhat.config.ts" local config_var=userConfig - local compile_only_presets=( - ir-optimize-evm+yul # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115 - ) + local compile_only_presets=() local settings_presets=( "${compile_only_presets[@]}" #ir-no-optimize # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + ir-optimize-evm+yul legacy-no-optimize legacy-optimize-evm-only legacy-optimize-evm+yul @@ -70,6 +69,11 @@ function gnosis_safe_test # TODO: Remove this when Hardhat implements them (https://github.com/nomiclabs/hardhat/issues/2051). sed -i "s|\(it\)\(('should revert if called directly', async () => {\)|\1.skip\2|g" test/handlers/CompatibilityFallbackHandler.spec.ts + # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables + # them for other presets but that's fine - we want same code run for benchmarks to be comparable. + # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115). + sed -i "s|\(it\)\(('should not allow to call setup on singleton'\)|\1.skip\2|g" test/core/GnosisSafe.Setup.spec.ts + neutralize_package_lock neutralize_package_json_hooks force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" diff --git a/test/externalTests/prb-math.sh b/test/externalTests/prb-math.sh index 01bdb76f1..1d0a0dd86 100755 --- a/test/externalTests/prb-math.sh +++ b/test/externalTests/prb-math.sh @@ -43,13 +43,12 @@ function prb_math_test local config_file="hardhat.config.ts" local config_var="config" - local compile_only_presets=( - ir-optimize-evm+yul # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115 - ) + local compile_only_presets=() local settings_presets=( "${compile_only_presets[@]}" #ir-no-optimize # Compilation fails with "YulException: Variable var_y_1960 is 8 slot(s) too deep inside the stack." #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_y_1960 is 8 slot(s) too deep inside the stack." + ir-optimize-evm+yul legacy-optimize-evm-only legacy-optimize-evm+yul legacy-no-optimize @@ -68,6 +67,26 @@ function prb_math_test # yarn.lock. Remove the config to restore Yarn 1.x. rm .yarnrc.yml + # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables + # them for other presets but that's fine - we want same code run for benchmarks to be comparable. + # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115). + pushd test/contracts/prbMathUd60x18/pure/ + sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts + sed -i 's|context(\("when the sum does not overflow"\)|context.skip(\1|g' add.test.ts + sed -i 's|context(\("when both operands are zero"\)|context.skip(\1|g' avg.test.ts + sed -i 's|context(\("when one operand is zero and the other is not zero"\)|context.skip(\1|g' avg.test.ts + sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts + sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts + popd + pushd test/contracts/prbMathSd59x18/pure/ + sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts + sed -i 's|context(\("when the sum underflows"\)|context.skip(\1|g' add.test.ts + sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts + sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts + sed -i 's|context(\("when the difference underflows"\)|context.skip(\1|g' sub.test.ts + sed -i 's|context(\("when the difference overflows"\)|context.skip(\1|g' sub.test.ts + popd + neutralize_package_lock neutralize_package_json_hooks force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" diff --git a/test/externalTests/zeppelin.sh b/test/externalTests/zeppelin.sh index 4094c6936..ab79a8775 100755 --- a/test/externalTests/zeppelin.sh +++ b/test/externalTests/zeppelin.sh @@ -41,13 +41,12 @@ function zeppelin_test local ref="master" local config_file="hardhat.config.js" - local compile_only_presets=( - ir-optimize-evm+yul # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115 - ) + local compile_only_presets=() local settings_presets=( "${compile_only_presets[@]}" #ir-no-optimize # Compilation fails with "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." + ir-optimize-evm+yul legacy-no-optimize legacy-optimize-evm-only legacy-optimize-evm+yul @@ -59,6 +58,19 @@ function zeppelin_test setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH" download_project "$repo" "$ref_type" "$ref" "$DIR" + # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables + # them for other presets but that's fine - we want same code run for benchmarks to be comparable. + # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115). + pushd test/utils/ + sed -i "s|it(\('reverts \)|it.skip(\1|g" math/SafeMath.test.js + sed -i "s|it(\('reverts \)|it.skip(\1|g" math/SignedSafeMath.test.js + sed -i "s|it(\('reverts \)|it.skip(\1|g" structs/EnumerableSet.behavior.js + popd + + # In some cases Hardhat does not detect revert reasons properly via IR. + # TODO: Remove this when https://github.com/NomicFoundation/hardhat/issues/2453 gets fixed. + sed -i "s|it(\('reverts if the current value is 0'\)|it.skip(\1|g" test/utils/Counters.test.js + neutralize_package_json_hooks force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")"