Benchmark report collector job + summary

.circleci/config.yml

@@ -1216,6 +1216,23 @@ jobs:
             - reports/externalTests/
       - gitter_notify_failure_unless_pr
 
+  c_ext_benchmarks:
+    <<: *base_node_small
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Combine benchmark reports
+          command: cat reports/externalTests/benchmark-*.json | scripts/externalTests/merge_benchmarks.sh > reports/externalTests/all-benchmarks.json
+      - run:
+          name: Summarize reports
+          command: cat reports/externalTests/all-benchmarks.json | scripts/externalTests/summarize_benchmarks.sh > reports/externalTests/summarized-benchmarks.json
+      - store_artifacts:
+          path: reports/externalTests/all-benchmarks.json
+      - store_artifacts:
+          path: reports/externalTests/summarized-benchmarks.json
+
   b_win: &b_win
     <<: *base_win_powershell_large
     steps:
@@ -1466,6 +1483,24 @@ workflows:
       - t_ems_ext: *job_native_test_ext_prb_math
       - t_ems_ext: *job_native_test_ext_elementfi
 
+      - c_ext_benchmarks:
+          <<: *workflow_trigger_on_tags
+          requires:
+            - t_ems_compile_ext_colony
+            - t_native_compile_ext_gnosis
+            - t_native_test_ext_gnosis_v2
+            - t_native_test_ext_zeppelin
+            - t_native_test_ext_ens
+            - t_native_test_ext_trident
+            - t_native_test_ext_euler
+            - t_native_test_ext_yield_liquidator
+            - t_native_test_ext_bleeps
+            - t_native_test_ext_pool_together
+            - t_native_test_ext_perpetual_pools
+            - t_native_test_ext_uniswap
+            - t_native_test_ext_prb_math
+            - t_native_test_ext_elementfi
+
       # Windows build and tests
       - b_win: *workflow_trigger_on_tags
       - b_win_release: *workflow_trigger_on_tags

scripts/externalTests/merge_benchmarks.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# ------------------------------------------------------------------------------
+# Reads multiple individual benchmark reports produced by scripts from
+# test/externalTests/ from standard input and creates a combined report.
+#
+# Usage:
+#    <script name>.sh < <CONCATENATED_REPORTS>
+#
+# CONCATENATED_REPORTS: JSON report files concatenated into a single stream (e.g. using cat).
+#
+# Example:
+#    cat reports/externalTests/benchmark-*.json | <script name>.sh
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2021 solidity contributors.
+#------------------------------------------------------------------------------
+
+set -euo pipefail
+
+# We expect a series of dicts of the form {"<project>": {"<preset>": {...}}}.
+# Unfortunately jq's built-in `add` filter can't handle nested dicts and
+# would just overwrite values sharing a project name instead of merging them.
+
+# This is done by first grouping the dicts into an array of the form
+# [
+#     [{"key": "<project1>", "value": {"<preset1>": {...}}}, {"key": "<project1>", "value": {"<preset2>": {...}}, ...],
+#     [{"key": "<project2>", "value": {"<preset1>": {...}}}, {"key": "<project2>", "value": {"<preset2>": {...}}, ...],
+#     ...
+# ]
+# and then using reduce() on each group sharing the same project name to convert it into a
+# dict having preset names as keys.
+jq --slurp  --indent 4 --sort-keys '
+    map(to_entries[]) |
+    group_by(.key) |
+    map({
+        (.[0].key): (
+            reduce (.[].value | to_entries[]) as {$key, $value} (
+                {}; . + {
+                    ($key): $value
+                }
+            )
+        )
+    }) |
+    add
+'

scripts/externalTests/summarize_benchmarks.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# ------------------------------------------------------------------------------
+# Reads a combined benchmark report from standard input and outputs an abbreviated
+# report containing only totals. Can handle individual reports coming directly
+# from scripts in test/externalTests/ as well as combined report from merge_benchmarks.sh.
+#
+# Usage:
+#    <script name>.sh < <CONCATENATED_REPORTS>
+#
+# CONCATENATED_REPORTS: JSON report files concatenated into a single stream (e.g. using cat).
+#
+# Example:
+#    cat reports/externalTests/benchmark-*.json | <script name>.sh
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2021 solidity contributors.
+#------------------------------------------------------------------------------
+
+set -euo pipefail
+
+REPO_ROOT=$(realpath "$(dirname "$0")/../..")
+
+# Iterates over presets in a dict of the form {"<project>": {"<preset>": {...}}} and for each
+# one preserves only the few keys with totals that we want to see in the summary.
+exec "${REPO_ROOT}/scripts/externalTests/merge_benchmarks.sh" | jq --indent 4 --sort-keys '
+    with_entries({
+        key: .key,
+        value: .value | with_entries({
+            key: .key,
+            value: {
+                bytecode_size: .value.total_bytecode_size,
+                method_gas: .value.gas.total_method_gas,
+                deployment_gas: .value.gas.total_deployment_gas,
+                version: .value.project.version
+            }
+        })
+    })
+'