From c6094bb0c28a6735b0868ad9373c0240404db881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 18 Dec 2021 00:36:30 +0100 Subject: [PATCH] externalTests: Benchmark reports --- .circleci/config.yml | 9 ++ test/externalTests.sh | 32 ++--- test/externalTests/bleeps.sh | 6 +- test/externalTests/colony.sh | 3 + test/externalTests/common.sh | 166 +++++++++++++++++++++++-- test/externalTests/elementfi.sh | 3 + test/externalTests/ens.sh | 3 + test/externalTests/euler.sh | 3 + test/externalTests/gnosis-v2.sh | 4 + test/externalTests/gnosis.sh | 4 + test/externalTests/perpetual-pools.sh | 3 + test/externalTests/pool-together.sh | 3 + test/externalTests/prb-math.sh | 7 +- test/externalTests/trident.sh | 3 + test/externalTests/uniswap.sh | 4 + test/externalTests/yield-liquidator.sh | 3 + test/externalTests/zeppelin.sh | 3 + 17 files changed, 232 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af58d830f..44837d831 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1205,6 +1205,15 @@ jobs: name: External <> tests (native) command: | test/externalTests/<>.sh native /tmp/workspace/solc/solc + - store_artifacts: + path: reports/externalTests/ + # persist_to_workspace fails if the directory does not exist and the test script will create + # it only if it actually has benchmark results. + - run: mkdir -p reports/externalTests/ + - persist_to_workspace: + root: . + paths: + - reports/externalTests/ - gitter_notify_failure_unless_pr b_win: &b_win diff --git a/test/externalTests.sh b/test/externalTests.sh index a5e52c852..b3b46460b 100755 --- a/test/externalTests.sh +++ b/test/externalTests.sh @@ -28,26 +28,26 @@ set -e -REPO_ROOT="$(dirname "$0")" - source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/..") + verify_input "$@" printTask "Running external tests..." -"$REPO_ROOT/externalTests/zeppelin.sh" "$@" -"$REPO_ROOT/externalTests/gnosis.sh" "$@" -"$REPO_ROOT/externalTests/gnosis-v2.sh" "$@" -"$REPO_ROOT/externalTests/colony.sh" "$@" -"$REPO_ROOT/externalTests/ens.sh" "$@" -"$REPO_ROOT/externalTests/trident.sh" "$@" -"$REPO_ROOT/externalTests/euler.sh" "$@" -"$REPO_ROOT/externalTests/yield-liquidator.sh" "$@" -"$REPO_ROOT/externalTests/bleeps.sh" "$@" -"$REPO_ROOT/externalTests/pool-together.sh" "$@" -"$REPO_ROOT/externalTests/perpetual-pools.sh" "$@" -"$REPO_ROOT/externalTests/uniswap.sh" "$@" -"$REPO_ROOT/externalTests/prb-math.sh" "$@" -"$REPO_ROOT/externalTests/elementfi.sh" "$@" +"{$REPO_ROOT}/test/externalTests/zeppelin.sh" "$@" +"{$REPO_ROOT}/test/externalTests/gnosis.sh" "$@" +"{$REPO_ROOT}/test/externalTests/gnosis-v2.sh" "$@" +"{$REPO_ROOT}/test/externalTests/colony.sh" "$@" +"{$REPO_ROOT}/test/externalTests/ens.sh" "$@" +"{$REPO_ROOT}/test/externalTests/trident.sh" "$@" +"{$REPO_ROOT}/test/externalTests/euler.sh" "$@" +"{$REPO_ROOT}/test/externalTests/yield-liquidator.sh" "$@" +"{$REPO_ROOT}/test/externalTests/bleeps.sh" "$@" +"{$REPO_ROOT}/test/externalTests/pool-together.sh" "$@" +"{$REPO_ROOT}/test/externalTests/perpetual-pools.sh" "$@" +"{$REPO_ROOT}/test/externalTests/uniswap.sh" "$@" +"{$REPO_ROOT}/test/externalTests/prb-math.sh" "$@" +"{$REPO_ROOT}/test/externalTests/elementfi.sh" "$@" diff --git a/test/externalTests/bleeps.sh b/test/externalTests/bleeps.sh index 561200e21..bd3b0255b 100755 --- a/test/externalTests/bleeps.sh +++ b/test/externalTests/bleeps.sh @@ -24,13 +24,16 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" SELECTED_PRESETS="$3" function compile_fn { npm run compile; } -function test_fn { npm run test; } +# NOTE: `npm run test` runs `mocha` which seems to disable the gas reporter. +function test_fn { HARDHAT_DEPLOY_FIXTURE=true npx --no hardhat --no-compile test; } function bleeps_test { @@ -87,6 +90,7 @@ function bleeps_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat bleeps "$repo" "$preset" done popd diff --git a/test/externalTests/colony.sh b/test/externalTests/colony.sh index d65989953..e23ce1135 100755 --- a/test/externalTests/colony.sh +++ b/test/externalTests/colony.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -73,6 +75,7 @@ function colony_test for preset in $SELECTED_PRESETS; do truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report truffle colony "$repo" "$preset" done } diff --git a/test/externalTests/common.sh b/test/externalTests/common.sh index cd0afb8a4..8f7b09eb5 100644 --- a/test/externalTests/common.sh +++ b/test/externalTests/common.sh @@ -20,7 +20,7 @@ #------------------------------------------------------------------------------ set -e -# Requires "${REPO_ROOT}/scripts/common.sh" to be included before. +# Requires $REPO_ROOT to be defined and "${REPO_ROOT}/scripts/common.sh" to be included before. CURRENT_EVM_VERSION=london @@ -207,9 +207,19 @@ function force_truffle_compiler_settings echo "Compiler version (full): ${SOLCVERSION}" echo "-------------------------------------" - # Forcing the settings should always work by just overwriting the solc object. Forcing them by using a - # dedicated settings objects should only be the fallback. - echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$preset" "$evm_version");" >> "$config_file" + local compiler_settings gas_reporter_settings + compiler_settings=$(truffle_compiler_settings "$solc_path" "$preset" "$evm_version") + gas_reporter_settings=$(eth_gas_reporter_settings "$preset") + + { + echo "require('eth-gas-reporter');" + echo "module.exports['mocha'] = {" + echo " reporter: 'eth-gas-reporter'," + echo " reporterOptions: ${gas_reporter_settings}" + echo "};" + + echo "module.exports['compilers'] = ${compiler_settings};" + } >> "$config_file" } function name_hardhat_default_export @@ -278,16 +288,21 @@ function force_hardhat_compiler_settings echo "Compiler version (full): ${SOLCVERSION}" echo "-------------------------------------" - local settings - settings=$(hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version") + local compiler_settings gas_reporter_settings + compiler_settings=$(hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version") + gas_reporter_settings=$(eth_gas_reporter_settings "$preset") if [[ $config_file == *\.js ]]; then [[ $config_var_name == "" ]] || assertFail - echo "module.exports['solidity'] = ${settings}" >> "$config_file" + echo "require('hardhat-gas-reporter');" + echo "module.exports.gasReporter = ${gas_reporter_settings};" + echo "module.exports.solidity = ${compiler_settings};" else [[ $config_file == *\.ts ]] || assertFail [[ $config_var_name != "" ]] || assertFail - echo "${config_var_name}.solidity = {compilers: [${settings}]}" >> "$config_file" - fi + echo 'import "hardhat-gas-reporter";' + echo "${config_var_name}.gasReporter = ${gas_reporter_settings};" + echo "${config_var_name}.solidity = {compilers: [${compiler_settings}]};" + fi >> "$config_file" } function truffle_verify_compiler_version @@ -368,6 +383,21 @@ function replace_global_solc export PATH="$PWD:$PATH" } +function eth_gas_reporter_settings +{ + local preset="$1" + + echo "{" + echo " enabled: true," + echo " gasPrice: 1," # Gas price does not matter to us at all. Set to whatever to avoid API call. + echo " noColors: true," + echo " showTimeSpent: false," # We're not interested in test timing + echo " onlyCalledMethods: true," # Exclude entries with no gas for shorter report + echo " showMethodSig: true," # Should make diffs more stable if there are overloaded functions + echo " outputFile: \"$(gas_report_path "$preset")\"" + echo "}" +} + function truffle_compiler_settings { local solc_path="$1" @@ -495,3 +525,121 @@ function external_test rm -rf "$DIR" echo "Done." } + +function gas_report_path +{ + local preset="$1" + + echo "${DIR}/gas-report-${preset}.rst" +} + +function gas_report_to_json +{ + cat - | "${REPO_ROOT}/scripts/externalTests/parse_eth_gas_report.py" | jq '{gas: .}' +} + +function detect_hardhat_artifact_dir +{ + if [[ -e build/ && -e artifacts/ ]]; then + fail "Cannot determine Hardhat artifact location. Both build/ and artifacts/ exist" + elif [[ -e build/ ]]; then + echo -n build/artifacts + elif [[ -e artifacts/ ]]; then + echo -n artifacts + else + fail "Hardhat build artifacts not found." + fi +} + +function bytecode_size_json_from_truffle_artifacts +{ + # NOTE: The output of this function is a series of concatenated JSON dicts rather than a list. + + for artifact in build/contracts/*.json; do + if [[ $(jq '. | has("unlinked_binary")' "$artifact") == false ]]; then + # Each artifact represents compilation output for a single contract. Some top-level keys contain + # bits of Standard JSON output while others are generated by Truffle. Process it into a dict + # of the form `{"": {"": }}`. + # NOTE: The `bytecode` field starts with 0x, which is why we subtract 1 from size. + jq '{ + (.ast.absolutePath): { + (.contractName): (.bytecode | length / 2 - 1) + } + }' "$artifact" + fi + done +} + +function bytecode_size_json_from_hardhat_artifacts +{ + # NOTE: The output of this function is a series of concatenated JSON dicts rather than a list. + + for artifact in "$(detect_hardhat_artifact_dir)"/build-info/*.json; do + # Each artifact contains Standard JSON output under the `output` key. + # Process it into a dict of the form `{"": {"": }}`, + # Note that one Hardhat artifact often represents multiple input files. + jq '.output.contracts | to_entries[] | { + "\(.key)": .value | to_entries[] | { + "\(.key)": (.value.evm.bytecode.object | length / 2) + } + }' "$artifact" + done +} + +function combine_artifact_json +{ + # Combine all dicts into a list with `jq --slurp` and then use `reduce` to merge them into one + # big dict with keys of the form `":"`. Then run jq again to filter out items + # with zero size and put the rest under under a top-level `bytecode_size` key. Also add another + # key with total bytecode size. + # NOTE: The extra inner `bytecode_size` key is there only to make diffs more readable. + cat - | + jq --slurp 'reduce (.[] | to_entries[]) as {$key, $value} ({}; . + { + ($key + ":" + ($value | to_entries[].key)): { + bytecode_size: $value | to_entries[].value + } + })' | + jq --indent 4 --sort-keys '{ + bytecode_size: [. | to_entries[] | select(.value.bytecode_size > 0)] | from_entries, + total_bytecode_size: (reduce (. | to_entries[]) as {$key, $value} (0; . + $value.bytecode_size)) + }' +} + +function project_info_json +{ + local project_url="$1" + + echo "{" + echo " \"project\": {" + # NOTE: Given that we clone with `--depth 1`, we'll only get useful output out of `git describe` + # if we directly check out a tag. Still better than nothing. + echo " \"version\": \"$(git describe --always)\"," + echo " \"commit\": \"$(git rev-parse HEAD)\"," + echo " \"url\": \"${project_url}\"" + echo " }" + echo "}" +} + +function store_benchmark_report +{ + local framework="$1" + local project_name="$2" + local project_url="$3" + local preset="$4" + + [[ $framework == truffle || $framework == hardhat ]] || assertFail + [[ " ${AVAILABLE_PRESETS[*]} " == *" $preset "* ]] || assertFail + + local report_dir="${REPO_ROOT}/reports/externalTests" + local output_file="${report_dir}/benchmark-${project_name}-${preset}.json" + mkdir -p "$report_dir" + + { + if [[ -e $(gas_report_path "$preset") ]]; then + gas_report_to_json < "$(gas_report_path "$preset")" + fi + + "bytecode_size_json_from_${framework}_artifacts" | combine_artifact_json + project_info_json "$project_url" + } | jq --slurp "{\"${project_name}\": {\"${preset}\": add}}" --indent 4 --sort-keys > "$output_file" +} diff --git a/test/externalTests/elementfi.sh b/test/externalTests/elementfi.sh index f52a13ca6..3641eac2b 100755 --- a/test/externalTests/elementfi.sh +++ b/test/externalTests/elementfi.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -96,6 +98,7 @@ function elementfi_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat elementfi "$repo" "$preset" done } diff --git a/test/externalTests/ens.sh b/test/externalTests/ens.sh index d3fcb0b13..734c5d8ae 100755 --- a/test/externalTests/ens.sh +++ b/test/externalTests/ens.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function ens_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report hardhat ens "$repo" "$preset" done } diff --git a/test/externalTests/euler.sh b/test/externalTests/euler.sh index 485d7ae56..b3b505051 100755 --- a/test/externalTests/euler.sh +++ b/test/externalTests/euler.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function euler_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report hardhat euler "$repo" "$preset" done } diff --git a/test/externalTests/gnosis-v2.sh b/test/externalTests/gnosis-v2.sh index a48dddf9c..6b0915b6a 100755 --- a/test/externalTests/gnosis-v2.sh +++ b/test/externalTests/gnosis-v2.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -65,12 +67,14 @@ function gnosis_safe_test neutralize_package_json_hooks force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$(first_word "$SELECTED_PRESETS")" npm install --package-lock + npm install eth-gas-reporter replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc/dist" for preset in $SELECTED_PRESETS; do truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report truffle gnosis2 "$repo" "$preset" done } diff --git a/test/externalTests/gnosis.sh b/test/externalTests/gnosis.sh index 1b430a9ae..82d1892f2 100755 --- a/test/externalTests/gnosis.sh +++ b/test/externalTests/gnosis.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -63,12 +65,14 @@ function gnosis_safe_test neutralize_package_json_hooks force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$(first_word "$SELECTED_PRESETS")" npm install --package-lock + npm install eth-gas-reporter replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc/dist" for preset in $SELECTED_PRESETS; do truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report truffle gnosis "$repo" "$preset" done } diff --git a/test/externalTests/perpetual-pools.sh b/test/externalTests/perpetual-pools.sh index e1ee9af8f..8f90ed0b4 100755 --- a/test/externalTests/perpetual-pools.sh +++ b/test/externalTests/perpetual-pools.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function perpetual_pools_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat perpetual-pools "$repo" "$preset" done } diff --git a/test/externalTests/pool-together.sh b/test/externalTests/pool-together.sh index a7f5933c0..b24cd6f4b 100755 --- a/test/externalTests/pool-together.sh +++ b/test/externalTests/pool-together.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -72,6 +74,7 @@ function pool_together_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat pool-together "$repo" "$preset" done } diff --git a/test/externalTests/prb-math.sh b/test/externalTests/prb-math.sh index 53fad2c25..01bdb76f1 100755 --- a/test/externalTests/prb-math.sh +++ b/test/externalTests/prb-math.sh @@ -24,13 +24,16 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" SELECTED_PRESETS="$3" function compile_fn { yarn compile; } -function test_fn { yarn test; } +# NOTE: `yarn test` runs `mocha` which seems to disable the gas reporter. +function test_fn { npx --no hardhat --no-compile test; } function prb_math_test { @@ -70,11 +73,13 @@ function prb_math_test force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")" "$config_var" yarn install --no-lock-file + yarn add hardhat-gas-reporter replace_version_pragmas for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat prb-math "$repo" "$preset" done } diff --git a/test/externalTests/trident.sh b/test/externalTests/trident.sh index 8e6f35ae3..5e230560d 100755 --- a/test/externalTests/trident.sh +++ b/test/externalTests/trident.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -92,6 +94,7 @@ function trident_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat trident "$repo" "$preset" done } diff --git a/test/externalTests/uniswap.sh b/test/externalTests/uniswap.sh index 96a5d2b63..2f94c6abd 100755 --- a/test/externalTests/uniswap.sh +++ b/test/externalTests/uniswap.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -73,11 +75,13 @@ function uniswap_test yarn add @ethereumjs/tx@3.1.3 yarn install + yarn add hardhat-gas-reporter replace_version_pragmas for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat uniswap "$repo" "$preset" done } diff --git a/test/externalTests/yield-liquidator.sh b/test/externalTests/yield-liquidator.sh index 6712941e7..1b5b9f146 100755 --- a/test/externalTests/yield-liquidator.sh +++ b/test/externalTests/yield-liquidator.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function yield_liquidator_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat yield_liquidator "$repo" "$preset" done } diff --git a/test/externalTests/zeppelin.sh b/test/externalTests/zeppelin.sh index d365dfe2a..4094c6936 100755 --- a/test/externalTests/zeppelin.sh +++ b/test/externalTests/zeppelin.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -66,6 +68,7 @@ function zeppelin_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report hardhat zeppelin "$repo" "$preset" done }