externalTests: Benchmark reports

.circleci/config.yml

@@ -1205,6 +1205,15 @@ jobs:
                 name: External <<parameters.project>> tests (native)
                 command: |
                   test/externalTests/<<parameters.project>>.sh native /tmp/workspace/solc/solc
+      - store_artifacts:
+          path: reports/externalTests/
+      # persist_to_workspace fails if the directory does not exist and the test script will create
+      # it only if it actually has benchmark results.
+      - run: mkdir -p reports/externalTests/
+      - persist_to_workspace:
+          root: .
+          paths:
+            - reports/externalTests/
       - gitter_notify_failure_unless_pr
 
   b_win: &b_win

test/externalTests.sh

@@ -28,26 +28,26 @@
 
 set -e
 
-REPO_ROOT="$(dirname "$0")"
-
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/..")
+
 verify_input "$@"
 
 printTask "Running external tests..."
 
-"$REPO_ROOT/externalTests/zeppelin.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/zeppelin.sh" "$@"
-"$REPO_ROOT/externalTests/gnosis.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/gnosis.sh" "$@"
-"$REPO_ROOT/externalTests/gnosis-v2.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/gnosis-v2.sh" "$@"
-"$REPO_ROOT/externalTests/colony.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/colony.sh" "$@"
-"$REPO_ROOT/externalTests/ens.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/ens.sh" "$@"
-"$REPO_ROOT/externalTests/trident.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/trident.sh" "$@"
-"$REPO_ROOT/externalTests/euler.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/euler.sh" "$@"
-"$REPO_ROOT/externalTests/yield-liquidator.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/yield-liquidator.sh" "$@"
-"$REPO_ROOT/externalTests/bleeps.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/bleeps.sh" "$@"
-"$REPO_ROOT/externalTests/pool-together.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/pool-together.sh" "$@"
-"$REPO_ROOT/externalTests/perpetual-pools.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/perpetual-pools.sh" "$@"
-"$REPO_ROOT/externalTests/uniswap.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/uniswap.sh" "$@"
-"$REPO_ROOT/externalTests/prb-math.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/prb-math.sh" "$@"
-"$REPO_ROOT/externalTests/elementfi.sh" "$@"
+"{$REPO_ROOT}/test/externalTests/elementfi.sh" "$@"

test/externalTests/bleeps.sh

@@ -24,13 +24,16 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
 SELECTED_PRESETS="$3"
 
 function compile_fn { npm run compile; }
-function test_fn { npm run test; }
+# NOTE: `npm run test` runs `mocha` which seems to disable the gas reporter.
+function test_fn { HARDHAT_DEPLOY_FIXTURE=true npx --no hardhat --no-compile test; }
 
 function bleeps_test
 {
@@ -87,6 +90,7 @@ function bleeps_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat bleeps "$repo" "$preset"
     done
 
     popd

test/externalTests/colony.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -73,6 +75,7 @@ function colony_test
 
     for preset in $SELECTED_PRESETS; do
         truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
+        store_benchmark_report truffle colony "$repo" "$preset"
     done
 }
 

test/externalTests/common.sh

@@ -20,7 +20,7 @@
 #------------------------------------------------------------------------------
 set -e
 
-# Requires "${REPO_ROOT}/scripts/common.sh" to be included before.
+# Requires $REPO_ROOT to be defined and "${REPO_ROOT}/scripts/common.sh" to be included before.
 
 CURRENT_EVM_VERSION=london
 
@@ -207,9 +207,19 @@ function force_truffle_compiler_settings
     echo "Compiler version (full): ${SOLCVERSION}"
     echo "-------------------------------------"
 
-    # Forcing the settings should always work by just overwriting the solc object. Forcing them by using a
+    local compiler_settings gas_reporter_settings
-    # dedicated settings objects should only be the fallback.
+    compiler_settings=$(truffle_compiler_settings "$solc_path" "$preset" "$evm_version")
-    echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$preset" "$evm_version");" >> "$config_file"
+    gas_reporter_settings=$(eth_gas_reporter_settings "$preset")
+
+    {
+        echo "require('eth-gas-reporter');"
+        echo "module.exports['mocha'] = {"
+        echo "    reporter: 'eth-gas-reporter',"
+        echo "    reporterOptions: ${gas_reporter_settings}"
+        echo "};"
+
+        echo "module.exports['compilers'] = ${compiler_settings};"
+    } >> "$config_file"
 }
 
 function name_hardhat_default_export
@@ -278,16 +288,21 @@ function force_hardhat_compiler_settings
     echo "Compiler version (full): ${SOLCVERSION}"
     echo "-------------------------------------"
 
-    local settings
+    local compiler_settings gas_reporter_settings
-    settings=$(hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version")
+    compiler_settings=$(hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version")
+    gas_reporter_settings=$(eth_gas_reporter_settings "$preset")
     if [[ $config_file == *\.js ]]; then
         [[ $config_var_name == "" ]] || assertFail
-        echo "module.exports['solidity'] = ${settings}" >> "$config_file"
+        echo "require('hardhat-gas-reporter');"
+        echo "module.exports.gasReporter = ${gas_reporter_settings};"
+        echo "module.exports.solidity = ${compiler_settings};"
     else
         [[ $config_file == *\.ts ]] || assertFail
         [[ $config_var_name != "" ]] || assertFail
-        echo "${config_var_name}.solidity = {compilers: [${settings}]}"  >> "$config_file"
+        echo 'import "hardhat-gas-reporter";'
-    fi
+        echo "${config_var_name}.gasReporter = ${gas_reporter_settings};"
+        echo "${config_var_name}.solidity = {compilers: [${compiler_settings}]};"
+    fi >> "$config_file"
 }
 
 function truffle_verify_compiler_version
@@ -368,6 +383,21 @@ function replace_global_solc
     export PATH="$PWD:$PATH"
 }
 
+function eth_gas_reporter_settings
+{
+    local preset="$1"
+
+    echo "{"
+    echo "    enabled: true,"
+    echo "    gasPrice: 1,"                           # Gas price does not matter to us at all. Set to whatever to avoid API call.
+    echo "    noColors: true,"
+    echo "    showTimeSpent: false,"                  # We're not interested in test timing
+    echo "    onlyCalledMethods: true,"               # Exclude entries with no gas for shorter report
+    echo "    showMethodSig: true,"                   # Should make diffs more stable if there are overloaded functions
+    echo "    outputFile: \"$(gas_report_path "$preset")\""
+    echo "}"
+}
+
 function truffle_compiler_settings
 {
     local solc_path="$1"
@@ -495,3 +525,121 @@ function external_test
     rm -rf "$DIR"
     echo "Done."
 }
+
+function gas_report_path
+{
+    local preset="$1"
+
+    echo "${DIR}/gas-report-${preset}.rst"
+}
+
+function gas_report_to_json
+{
+    cat - | "${REPO_ROOT}/scripts/externalTests/parse_eth_gas_report.py" | jq '{gas: .}'
+}
+
+function detect_hardhat_artifact_dir
+{
+    if [[ -e build/ && -e artifacts/ ]]; then
+        fail "Cannot determine Hardhat artifact location. Both build/ and artifacts/ exist"
+    elif [[ -e build/ ]]; then
+        echo -n build/artifacts
+    elif [[ -e artifacts/ ]]; then
+        echo -n artifacts
+    else
+        fail "Hardhat build artifacts not found."
+    fi
+}
+
+function bytecode_size_json_from_truffle_artifacts
+{
+    # NOTE: The output of this function is a series of concatenated JSON dicts rather than a list.
+
+    for artifact in build/contracts/*.json; do
+        if [[ $(jq '. | has("unlinked_binary")' "$artifact") == false ]]; then
+            # Each artifact represents compilation output for a single contract. Some top-level keys contain
+            # bits of Standard JSON output while others are generated by Truffle. Process it into a dict
+            # of the form `{"<file>": {"<contract>": <size>}}`.
+            # NOTE: The `bytecode` field starts with 0x, which is why we subtract 1 from size.
+            jq '{
+                (.ast.absolutePath): {
+                    (.contractName): (.bytecode | length / 2 - 1)
+                }
+            }' "$artifact"
+        fi
+    done
+}
+
+function bytecode_size_json_from_hardhat_artifacts
+{
+    # NOTE: The output of this function is a series of concatenated JSON dicts rather than a list.
+
+    for artifact in "$(detect_hardhat_artifact_dir)"/build-info/*.json; do
+        # Each artifact contains Standard JSON output under the `output` key.
+        # Process it into a dict of the form `{"<file>": {"<contract>": <size>}}`,
+        # Note that one Hardhat artifact often represents multiple input files.
+        jq '.output.contracts | to_entries[] | {
+            "\(.key)": .value | to_entries[] | {
+                "\(.key)": (.value.evm.bytecode.object | length / 2)
+            }
+        }' "$artifact"
+    done
+}
+
+function combine_artifact_json
+{
+    # Combine all dicts into a list with `jq --slurp` and then use `reduce` to merge them into one
+    # big dict with keys of the form `"<file>:<contract>"`. Then run jq again to filter out items
+    # with zero size and put the rest under under a top-level `bytecode_size` key. Also add another
+    # key with total bytecode size.
+    # NOTE: The extra inner `bytecode_size` key is there only to make diffs more readable.
+    cat - |
+        jq --slurp 'reduce (.[] | to_entries[]) as {$key, $value} ({}; . + {
+            ($key + ":" + ($value | to_entries[].key)): {
+                bytecode_size: $value | to_entries[].value
+            }
+        })' |
+        jq --indent 4 --sort-keys '{
+            bytecode_size: [. | to_entries[] | select(.value.bytecode_size > 0)] | from_entries,
+            total_bytecode_size: (reduce (. | to_entries[]) as {$key, $value} (0; . + $value.bytecode_size))
+        }'
+}
+
+function project_info_json
+{
+    local project_url="$1"
+
+    echo "{"
+    echo "    \"project\": {"
+    # NOTE: Given that we clone with `--depth 1`, we'll only get useful output out of `git describe`
+    # if we directly check out a tag. Still better than nothing.
+    echo "        \"version\": \"$(git describe --always)\","
+    echo "        \"commit\": \"$(git rev-parse HEAD)\","
+    echo "        \"url\": \"${project_url}\""
+    echo "    }"
+    echo "}"
+}
+
+function store_benchmark_report
+{
+    local framework="$1"
+    local project_name="$2"
+    local project_url="$3"
+    local preset="$4"
+
+    [[ $framework == truffle || $framework == hardhat ]] || assertFail
+    [[ " ${AVAILABLE_PRESETS[*]} " == *" $preset "* ]] || assertFail
+
+    local report_dir="${REPO_ROOT}/reports/externalTests"
+    local output_file="${report_dir}/benchmark-${project_name}-${preset}.json"
+    mkdir -p "$report_dir"
+
+    {
+        if [[ -e $(gas_report_path "$preset") ]]; then
+            gas_report_to_json < "$(gas_report_path "$preset")"
+        fi
+
+        "bytecode_size_json_from_${framework}_artifacts" | combine_artifact_json
+        project_info_json "$project_url"
+    } | jq --slurp "{\"${project_name}\": {\"${preset}\": add}}" --indent 4 --sort-keys > "$output_file"
+}

test/externalTests/elementfi.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -96,6 +98,7 @@ function elementfi_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat elementfi "$repo" "$preset"
     done
 }
 

test/externalTests/ens.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -68,6 +70,7 @@ function ens_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
+        store_benchmark_report hardhat ens "$repo" "$preset"
     done
 }
 

test/externalTests/euler.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -68,6 +70,7 @@ function euler_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
+        store_benchmark_report hardhat euler "$repo" "$preset"
     done
 }
 

test/externalTests/gnosis-v2.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -65,12 +67,14 @@ function gnosis_safe_test
     neutralize_package_json_hooks
     force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$(first_word "$SELECTED_PRESETS")"
     npm install --package-lock
+    npm install eth-gas-reporter
 
     replace_version_pragmas
     [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc/dist"
 
     for preset in $SELECTED_PRESETS; do
         truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
+        store_benchmark_report truffle gnosis2 "$repo" "$preset"
     done
 }
 

test/externalTests/gnosis.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -63,12 +65,14 @@ function gnosis_safe_test
     neutralize_package_json_hooks
     force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$(first_word "$SELECTED_PRESETS")"
     npm install --package-lock
+    npm install eth-gas-reporter
 
     replace_version_pragmas
     [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc/dist"
 
     for preset in $SELECTED_PRESETS; do
         truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
+        store_benchmark_report truffle gnosis "$repo" "$preset"
     done
 }
 

test/externalTests/perpetual-pools.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -68,6 +70,7 @@ function perpetual_pools_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat perpetual-pools "$repo" "$preset"
     done
 }
 

test/externalTests/pool-together.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -72,6 +74,7 @@ function pool_together_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat pool-together "$repo" "$preset"
     done
 }
 

test/externalTests/prb-math.sh

@@ -24,13 +24,16 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
 SELECTED_PRESETS="$3"
 
 function compile_fn { yarn compile; }
-function test_fn { yarn test; }
+# NOTE: `yarn test` runs `mocha` which seems to disable the gas reporter.
+function test_fn { npx --no hardhat --no-compile test; }
 
 function prb_math_test
 {
@@ -70,11 +73,13 @@ function prb_math_test
     force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"
     force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")" "$config_var"
     yarn install --no-lock-file
+    yarn add hardhat-gas-reporter
 
     replace_version_pragmas
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat prb-math "$repo" "$preset"
     done
 }
 

test/externalTests/trident.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -92,6 +94,7 @@ function trident_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat trident "$repo" "$preset"
     done
 }
 

test/externalTests/uniswap.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -73,11 +75,13 @@ function uniswap_test
     yarn add @ethereumjs/tx@3.1.3
 
     yarn install
+    yarn add hardhat-gas-reporter
 
     replace_version_pragmas
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat uniswap "$repo" "$preset"
     done
 }
 

test/externalTests/yield-liquidator.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -68,6 +70,7 @@ function yield_liquidator_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var"
+        store_benchmark_report hardhat yield_liquidator "$repo" "$preset"
     done
 }
 

test/externalTests/zeppelin.sh

@@ -24,6 +24,8 @@ set -e
 source scripts/common.sh
 source test/externalTests/common.sh
 
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
 verify_input "$@"
 BINARY_TYPE="$1"
 BINARY_PATH="$2"
@@ -66,6 +68,7 @@ function zeppelin_test
 
     for preset in $SELECTED_PRESETS; do
         hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
+        store_benchmark_report hardhat zeppelin "$repo" "$preset"
     done
 }