Merge remote-tracking branch 'origin/develop' into breaking

scripts/common.sh

@@ -246,3 +246,12 @@ function first_word
 
     echo "$words" | cut -d " " -f 1
 }
+
+# Function reads from stdin. Therefore it has to be used with pipes.
+function split_on_empty_lines_into_numbered_files
+{
+    path_prefix="${1}"
+    path_suffix="${2}"
+
+    awk -v RS= "{print > (\"${path_prefix}_\"NR \"${path_suffix}\")}"
+}

scripts/docs.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-#------------------------------------------------------------------------------
-# Bash script to build the Solidity Sphinx documentation locally.
-#
-# The documentation for solidity is hosted at:
-#
-#     https://docs.soliditylang.org
-#
-# ------------------------------------------------------------------------------
-# This file is part of solidity.
-#
-# solidity is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# solidity is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with solidity.  If not, see <http://www.gnu.org/licenses/>
-#
-# (c) 2016 solidity contributors.
-#------------------------------------------------------------------------------
-
-set -e
-cd docs
-pip3 install -r requirements.txt --upgrade --upgrade-strategy eager
-sphinx-build -nW -b html -d _build/doctrees . _build/html
-cd ..

scripts/externalTests/benchmark_diff.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from textwrap import dedent
+from typing import Any, Mapping, Optional, Set, Sequence, Union
+import json
+import sys
+
+
+class DiffMode(Enum):
+    IN_PLACE = 'inplace'
+    TABLE = 'table'
+
+
+class DifferenceStyle(Enum):
+    ABSOLUTE = 'absolute'
+    RELATIVE = 'relative'
+    HUMANIZED = 'humanized'
+
+
+class OutputFormat(Enum):
+    JSON = 'json'
+    CONSOLE = 'console'
+    MARKDOWN = 'markdown'
+
+
+DEFAULT_RELATIVE_PRECISION = 4
+
+DEFAULT_DIFFERENCE_STYLE = {
+    DiffMode.IN_PLACE: DifferenceStyle.ABSOLUTE,
+    DiffMode.TABLE: DifferenceStyle.HUMANIZED,
+}
+assert all(t in DiffMode for t in DEFAULT_DIFFERENCE_STYLE)
+assert all(d in DifferenceStyle for d in DEFAULT_DIFFERENCE_STYLE.values())
+
+DEFAULT_OUTPUT_FORMAT = {
+    DiffMode.IN_PLACE: OutputFormat.JSON,
+    DiffMode.TABLE: OutputFormat.CONSOLE,
+}
+assert all(m in DiffMode for m in DEFAULT_OUTPUT_FORMAT)
+assert all(o in OutputFormat for o in DEFAULT_OUTPUT_FORMAT.values())
+
+
+class ValidationError(Exception):
+    pass
+
+
+class CommandLineError(ValidationError):
+    pass
+
+
+class BenchmarkDiffer:
+    difference_style: DifferenceStyle
+    relative_precision: Optional[int]
+    output_format: OutputFormat
+
+    def __init__(
+        self,
+        difference_style: DifferenceStyle,
+        relative_precision: Optional[int],
+        output_format: OutputFormat,
+    ):
+        self.difference_style = difference_style
+        self.relative_precision = relative_precision
+        self.output_format = output_format
+
+    def run(self, before: Any, after: Any) -> Optional[Union[dict, str, int, float]]:
+        if not isinstance(before, dict) or not isinstance(after, dict):
+            return self._diff_scalars(before, after)
+
+        if before.get('version') != after.get('version'):
+            return self._humanize_diff('!V')
+
+        diff = {}
+        for key in (set(before) | set(after)) - {'version'}:
+            value_diff = self.run(before.get(key), after.get(key))
+            if value_diff not in [None, {}]:
+                diff[key] = value_diff
+
+        return diff
+
+    def _diff_scalars(self, before: Any, after: Any) -> Optional[Union[str, int, float, dict]]:
+        assert not isinstance(before, dict) or not isinstance(after, dict)
+
+        if before is None and after is None:
+            return {}
+        if before is None:
+            return self._humanize_diff('!B')
+        if after is None:
+            return self._humanize_diff('!A')
+        if not isinstance(before, (int, float)) or not isinstance(after, (int, float)):
+            return self._humanize_diff('!T')
+
+        number_diff = self._diff_numbers(before, after)
+        if self.difference_style != DifferenceStyle.HUMANIZED:
+            return number_diff
+
+        return self._humanize_diff(number_diff)
+
+    def _diff_numbers(self, value_before: Union[int, float], value_after: Union[int, float]) -> Union[str, int, float]:
+        diff: Union[str, int, float]
+
+        if self.difference_style == DifferenceStyle.ABSOLUTE:
+            diff = value_after - value_before
+            if isinstance(diff, float) and diff.is_integer():
+                diff = int(diff)
+
+            return diff
+
+        if value_before == 0:
+            if value_after > 0:
+                return '+INF'
+            elif value_after < 0:
+                return '-INF'
+            else:
+                return 0
+
+        diff = (value_after - value_before) / abs(value_before)
+        if self.relative_precision is not None:
+            rounded_diff = round(diff, self.relative_precision)
+            if rounded_diff == 0 and diff < 0:
+                diff = '-0'
+            elif rounded_diff == 0 and diff > 0:
+                diff = '+0'
+            else:
+                diff = rounded_diff
+
+        if isinstance(diff, float) and diff.is_integer():
+            diff = int(diff)
+
+        return diff
+
+    def _humanize_diff(self, diff: Union[str, int, float]) -> str:
+        def wrap(value: str, symbol: str):
+            return f"{symbol}{value}{symbol}"
+
+        markdown = (self.output_format == OutputFormat.MARKDOWN)
+
+        if isinstance(diff, str) and diff.startswith('!'):
+            return wrap(diff, '`' if markdown else '')
+
+        value: Union[str, int, float]
+        if isinstance(diff, (int, float)):
+            value = diff * 100
+            if isinstance(value, float) and self.relative_precision is not None:
+                # The multiplication can result in new significant digits appearing. We need to reround.
+                # NOTE: round() works fine with negative precision.
+                value = round(value, self.relative_precision - 2)
+                if isinstance(value, float) and value.is_integer():
+                    value = int(value)
+            suffix = ''
+            prefix = ''
+            if diff < 0:
+                prefix = ''
+                if markdown:
+                    suffix += ' ✅'
+            elif diff > 0:
+                prefix = '+'
+                if markdown:
+                    suffix += ' ❌'
+            important = (diff != 0)
+        else:
+            value = diff
+            important = False
+            prefix = ''
+            suffix = ''
+
+        return wrap(
+            wrap(
+                f"{prefix}{value}%{suffix}",
+                '`' if markdown else ''
+            ),
+            '**' if important and markdown else ''
+        )
+
+
+@dataclass(frozen=True)
+class DiffTable:
+    columns: Mapping[str, Sequence[Union[int, float, str]]]
+
+
+class DiffTableSet:
+    table_headers: Sequence[str]
+    row_headers: Sequence[str]
+    column_headers: Sequence[str]
+
+    # Cells is a nested dict rather than a 3D array so that conversion to JSON is straightforward
+    cells: Mapping[str, Mapping[str, Mapping[str, Union[int, float, str]]]] # preset -> project -> attribute
+
+    def __init__(self, diff: dict):
+        self.table_headers = sorted(self._find_all_preset_names(diff))
+        self.column_headers = sorted(self._find_all_attribute_names(diff))
+        self.row_headers = sorted(project for project in diff)
+
+        # All dimensions must have unique values
+        assert len(self.table_headers) == len(set(self.table_headers))
+        assert len(self.column_headers) == len(set(self.column_headers))
+        assert len(self.row_headers) == len(set(self.row_headers))
+
+        self.cells = {
+            preset: {
+                project: {
+                    attribute: self._cell_content(diff, project, preset, attribute)
+                    for attribute in self.column_headers
+                }
+                for project in self.row_headers
+            }
+            for preset in self.table_headers
+        }
+
+    def calculate_row_column_width(self) -> int:
+        return max(len(h) for h in self.row_headers)
+
+    def calculate_column_widths(self, table_header: str) -> Sequence[int]:
+        assert table_header in self.table_headers
+
+        return [
+            max(
+                len(column_header),
+                max(
+                    len(str(self.cells[table_header][row_header][column_header]))
+                    for row_header in self.row_headers
+                )
+            )
+            for column_header in self.column_headers
+        ]
+
+    @classmethod
+    def _find_all_preset_names(cls, diff: dict) -> Set[str]:
+        return {
+            preset
+            for project, project_diff in diff.items()
+            if isinstance(project_diff, dict)
+            for preset in project_diff
+        }
+
+    @classmethod
+    def _find_all_attribute_names(cls, diff: dict) -> Set[str]:
+        return {
+            attribute
+            for project, project_diff in diff.items()
+            if isinstance(project_diff, dict)
+            for preset, preset_diff in project_diff.items()
+            if isinstance(preset_diff, dict)
+            for attribute in preset_diff
+        }
+
+    @classmethod
+    def _cell_content(cls, diff: dict, project: str, preset: str, attribute: str) -> str:
+        assert project in diff
+
+        if isinstance(diff[project], str):
+            return diff[project]
+        if preset not in diff[project]:
+            return ''
+        if isinstance(diff[project][preset], str):
+            return diff[project][preset]
+        if attribute not in diff[project][preset]:
+            return ''
+
+        return diff[project][preset][attribute]
+
+
+class DiffTableFormatter:
+    LEGEND = dedent("""
+        `!V` = version mismatch
+        `!B` = no value in the "before" version
+        `!A` = no value in the "after" version
+        `!T` = one or both values were not numeric and could not be compared
+        `-0` = very small negative value rounded to zero
+        `+0` = very small positive value rounded to zero
+    """)
+
+    @classmethod
+    def run(cls, diff_table_set: DiffTableSet, output_format: OutputFormat):
+        if output_format == OutputFormat.JSON:
+            return json.dumps(diff_table_set.cells, indent=4, sort_keys=True)
+        else:
+            assert output_format in {OutputFormat.CONSOLE, OutputFormat.MARKDOWN}
+
+            output = ''
+            for table_header in diff_table_set.table_headers:
+                column_widths = ([
+                    diff_table_set.calculate_row_column_width(),
+                    *diff_table_set.calculate_column_widths(table_header)
+                ])
+
+                if output_format == OutputFormat.MARKDOWN:
+                    output += f'\n### `{table_header}`\n'
+                else:
+                    output += f'\n{table_header.upper()}\n'
+
+                if output_format == OutputFormat.CONSOLE:
+                    output += cls._format_separator_row(column_widths, output_format) + '\n'
+                output += cls._format_data_row(['project', *diff_table_set.column_headers], column_widths) + '\n'
+                output += cls._format_separator_row(column_widths, output_format) + '\n'
+
+                for row_header in diff_table_set.row_headers:
+                    row = [
+                        diff_table_set.cells[table_header][row_header][column_header]
+                        for column_header in diff_table_set.column_headers
+                    ]
+                    output += cls._format_data_row([row_header, *row], column_widths) + '\n'
+
+                if output_format == OutputFormat.CONSOLE:
+                    output += cls._format_separator_row(column_widths, output_format) + '\n'
+
+            if output_format == OutputFormat.MARKDOWN:
+                output += f'\n{cls.LEGEND}\n'
+            return output
+
+    @classmethod
+    def _format_separator_row(cls, widths: Sequence[int], output_format: OutputFormat):
+        assert output_format in {OutputFormat.CONSOLE, OutputFormat.MARKDOWN}
+
+        if output_format == OutputFormat.MARKDOWN:
+            return '|:' + ':|-'.join('-' * width for width in widths) + ':|'
+        else:
+            return '|-' + '-|-'.join('-' * width for width in widths) + '-|'
+
+    @classmethod
+    def _format_data_row(cls, cells: Sequence[Union[int, float, str]], widths: Sequence[int]):
+        assert len(cells) == len(widths)
+
+        return '| ' + ' | '.join(str(cell).rjust(width) for cell, width in zip(cells, widths)) + ' |'
+
+
+@dataclass(frozen=True)
+class CommandLineOptions:
+    diff_mode: DiffMode
+    report_before: Path
+    report_after: Path
+    difference_style: DifferenceStyle
+    relative_precision: int
+    output_format: OutputFormat
+
+
+def process_commandline() -> CommandLineOptions:
+    script_description = (
+        "Compares summarized benchmark reports and outputs JSON with the same structure but listing only differences. "
+        "Can also print the output as markdown table and format the values to make differences stand out more."
+    )
+
+    parser = ArgumentParser(description=script_description)
+    parser.add_argument(
+        dest='diff_mode',
+        choices=[m.value for m in DiffMode],
+        help=(
+            "Diff mode: "
+            f"'{DiffMode.IN_PLACE.value}' preserves input JSON structure and replace values with differences; "
+            f"'{DiffMode.TABLE.value}' creates a table assuming 3-level project/preset/attribute structure."
+        )
+    )
+    parser.add_argument(dest='report_before', help="Path to a JSON file containing original benchmark results.")
+    parser.add_argument(dest='report_after', help="Path to a JSON file containing new benchmark results.")
+    parser.add_argument(
+        '--style',
+        dest='difference_style',
+        choices=[s.value for s in DifferenceStyle],
+        help=(
+            "How to present numeric differences: "
+            f"'{DifferenceStyle.ABSOLUTE.value}' subtracts new from original; "
+            f"'{DifferenceStyle.RELATIVE.value}' also divides by the original; "
+            f"'{DifferenceStyle.HUMANIZED.value}' is like relative but value is a percentage and "
+            "positive/negative changes are emphasized. "
+            f"(default: '{DEFAULT_DIFFERENCE_STYLE[DiffMode.IN_PLACE]}' in '{DiffMode.IN_PLACE.value}' mode, "
+            f"'{DEFAULT_DIFFERENCE_STYLE[DiffMode.TABLE]}' in '{DiffMode.TABLE.value}' mode)"
+        )
+    )
+    # NOTE: Negative values are valid for precision. round() handles them in a sensible way.
+    parser.add_argument(
+        '--precision',
+        dest='relative_precision',
+        type=int,
+        default=DEFAULT_RELATIVE_PRECISION,
+        help=(
+            "Number of significant digits for relative differences. "
+            f"Note that with --style={DifferenceStyle.HUMANIZED.value} the rounding is applied "
+            "**before** converting the value to a percentage so you need to add 2. "
+            f"Has no effect when used together with --style={DifferenceStyle.ABSOLUTE.value}. "
+            f"(default: {DEFAULT_RELATIVE_PRECISION})"
+        )
+    )
+    parser.add_argument(
+        '--output-format',
+        dest='output_format',
+        choices=[o.value for o in OutputFormat],
+        help=(
+            "The format to use for the diff: "
+            f"'{OutputFormat.JSON.value}' is raw JSON; "
+            f"'{OutputFormat.CONSOLE.value}' is a table with human-readable values that will look good in the console output. "
+            f"'{OutputFormat.MARKDOWN.value}' is similar '{OutputFormat.CONSOLE.value}' but adjusted to "
+            "render as proper markdown and with extra elements (legend, emoji to make non-zero values stand out more, etc)."
+            f"(default: '{DEFAULT_OUTPUT_FORMAT[DiffMode.IN_PLACE]}' in '{DiffMode.IN_PLACE.value}' mode, "
+            f"'{DEFAULT_OUTPUT_FORMAT[DiffMode.TABLE]}' in '{DiffMode.TABLE.value}' mode)"
+        )
+    )
+
+    options = parser.parse_args()
+
+    if options.difference_style is not None:
+        difference_style = DifferenceStyle(options.difference_style)
+    else:
+        difference_style = DEFAULT_DIFFERENCE_STYLE[DiffMode(options.diff_mode)]
+
+    if options.output_format is not None:
+        output_format = OutputFormat(options.output_format)
+    else:
+        output_format = DEFAULT_OUTPUT_FORMAT[DiffMode(options.diff_mode)]
+
+    processed_options = CommandLineOptions(
+        diff_mode=DiffMode(options.diff_mode),
+        report_before=Path(options.report_before),
+        report_after=Path(options.report_after),
+        difference_style=difference_style,
+        relative_precision=options.relative_precision,
+        output_format=output_format,
+    )
+
+    if processed_options.diff_mode == DiffMode.IN_PLACE and processed_options.output_format != OutputFormat.JSON:
+        raise CommandLineError(
+            f"Only the '{OutputFormat.JSON.value}' output format is supported in the '{DiffMode.IN_PLACE.value}' mode."
+        )
+
+    return processed_options
+
+
+def main():
+    try:
+        options = process_commandline()
+
+        differ = BenchmarkDiffer(options.difference_style, options.relative_precision, options.output_format)
+        diff = differ.run(
+            json.loads(options.report_before.read_text('utf-8')),
+            json.loads(options.report_after.read_text('utf-8')),
+        )
+
+        if options.diff_mode == DiffMode.IN_PLACE:
+            print(json.dumps(diff, indent=4, sort_keys=True))
+        else:
+            assert options.diff_mode == DiffMode.TABLE
+            print(DiffTableFormatter.run(DiffTableSet(diff), options.output_format))
+
+        return 0
+    except CommandLineError as exception:
+        print(f"ERROR: {exception}", file=sys.stderr)
+        return 1
+
+if __name__ == "__main__":
+    sys.exit(main())