mirror of
https://github.com/ethereum/solidity
synced 2023-10-03 13:03:40 +00:00
benchmark_diff: Add table mode with support for json, console and markdown output
This commit is contained in:
parent
ee5e878ad7
commit
8c9856c52c
@ -4,19 +4,44 @@ from argparse import ArgumentParser
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Optional, Union
|
from textwrap import dedent
|
||||||
|
from typing import Any, Mapping, Optional, Set, Sequence, Union
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
class DiffMode(Enum):
|
||||||
|
IN_PLACE = 'inplace'
|
||||||
|
TABLE = 'table'
|
||||||
|
|
||||||
|
|
||||||
class DifferenceStyle(Enum):
|
class DifferenceStyle(Enum):
|
||||||
ABSOLUTE = 'absolute'
|
ABSOLUTE = 'absolute'
|
||||||
RELATIVE = 'relative'
|
RELATIVE = 'relative'
|
||||||
HUMANIZED = 'humanized'
|
HUMANIZED = 'humanized'
|
||||||
|
|
||||||
|
|
||||||
|
class OutputFormat(Enum):
|
||||||
|
JSON = 'json'
|
||||||
|
CONSOLE = 'console'
|
||||||
|
MARKDOWN = 'markdown'
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_RELATIVE_PRECISION = 4
|
DEFAULT_RELATIVE_PRECISION = 4
|
||||||
DEFAULT_DIFFERENCE_STYLE = DifferenceStyle.ABSOLUTE
|
|
||||||
|
DEFAULT_DIFFERENCE_STYLE = {
|
||||||
|
DiffMode.IN_PLACE: DifferenceStyle.ABSOLUTE,
|
||||||
|
DiffMode.TABLE: DifferenceStyle.HUMANIZED,
|
||||||
|
}
|
||||||
|
assert all(t in DiffMode for t in DEFAULT_DIFFERENCE_STYLE)
|
||||||
|
assert all(d in DifferenceStyle for d in DEFAULT_DIFFERENCE_STYLE.values())
|
||||||
|
|
||||||
|
DEFAULT_OUTPUT_FORMAT = {
|
||||||
|
DiffMode.IN_PLACE: OutputFormat.JSON,
|
||||||
|
DiffMode.TABLE: OutputFormat.CONSOLE,
|
||||||
|
}
|
||||||
|
assert all(m in DiffMode for m in DEFAULT_OUTPUT_FORMAT)
|
||||||
|
assert all(o in OutputFormat for o in DEFAULT_OUTPUT_FORMAT.values())
|
||||||
|
|
||||||
|
|
||||||
class ValidationError(Exception):
|
class ValidationError(Exception):
|
||||||
@ -30,14 +55,17 @@ class CommandLineError(ValidationError):
|
|||||||
class BenchmarkDiffer:
|
class BenchmarkDiffer:
|
||||||
difference_style: DifferenceStyle
|
difference_style: DifferenceStyle
|
||||||
relative_precision: Optional[int]
|
relative_precision: Optional[int]
|
||||||
|
output_format: OutputFormat
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
difference_style: DifferenceStyle,
|
difference_style: DifferenceStyle,
|
||||||
relative_precision: Optional[int],
|
relative_precision: Optional[int],
|
||||||
|
output_format: OutputFormat,
|
||||||
):
|
):
|
||||||
self.difference_style = difference_style
|
self.difference_style = difference_style
|
||||||
self.relative_precision = relative_precision
|
self.relative_precision = relative_precision
|
||||||
|
self.output_format = output_format
|
||||||
|
|
||||||
def run(self, before: Any, after: Any) -> Optional[Union[dict, str, int, float]]:
|
def run(self, before: Any, after: Any) -> Optional[Union[dict, str, int, float]]:
|
||||||
if not isinstance(before, dict) or not isinstance(after, dict):
|
if not isinstance(before, dict) or not isinstance(after, dict):
|
||||||
@ -106,8 +134,13 @@ class BenchmarkDiffer:
|
|||||||
return diff
|
return diff
|
||||||
|
|
||||||
def _humanize_diff(self, diff: Union[str, int, float]) -> str:
|
def _humanize_diff(self, diff: Union[str, int, float]) -> str:
|
||||||
|
def wrap(value: str, symbol: str):
|
||||||
|
return f"{symbol}{value}{symbol}"
|
||||||
|
|
||||||
|
markdown = (self.output_format == OutputFormat.MARKDOWN)
|
||||||
|
|
||||||
if isinstance(diff, str) and diff.startswith('!'):
|
if isinstance(diff, str) and diff.startswith('!'):
|
||||||
return diff
|
return wrap(diff, '`' if markdown else '')
|
||||||
|
|
||||||
value: Union[str, int, float]
|
value: Union[str, int, float]
|
||||||
if isinstance(diff, (int, float)):
|
if isinstance(diff, (int, float)):
|
||||||
@ -118,32 +151,209 @@ class BenchmarkDiffer:
|
|||||||
value = round(value, self.relative_precision - 2)
|
value = round(value, self.relative_precision - 2)
|
||||||
if isinstance(value, float) and value.is_integer():
|
if isinstance(value, float) and value.is_integer():
|
||||||
value = int(value)
|
value = int(value)
|
||||||
|
suffix = ''
|
||||||
prefix = ''
|
prefix = ''
|
||||||
if diff < 0:
|
if diff < 0:
|
||||||
prefix = ''
|
prefix = ''
|
||||||
|
if markdown:
|
||||||
|
suffix += ' ✅'
|
||||||
elif diff > 0:
|
elif diff > 0:
|
||||||
prefix = '+'
|
prefix = '+'
|
||||||
|
if markdown:
|
||||||
|
suffix += ' ❌'
|
||||||
|
important = (diff != 0)
|
||||||
else:
|
else:
|
||||||
value = diff
|
value = diff
|
||||||
|
important = False
|
||||||
prefix = ''
|
prefix = ''
|
||||||
|
suffix = ''
|
||||||
|
|
||||||
return f"{prefix}{value}%"
|
return wrap(
|
||||||
|
wrap(
|
||||||
|
f"{prefix}{value}%{suffix}",
|
||||||
|
'`' if markdown else ''
|
||||||
|
),
|
||||||
|
'**' if important and markdown else ''
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DiffTable:
|
||||||
|
columns: Mapping[str, Sequence[Union[int, float, str]]]
|
||||||
|
|
||||||
|
|
||||||
|
class DiffTableSet:
|
||||||
|
table_headers: Sequence[str]
|
||||||
|
row_headers: Sequence[str]
|
||||||
|
column_headers: Sequence[str]
|
||||||
|
|
||||||
|
# Cells is a nested dict rather than a 3D array so that conversion to JSON is straightforward
|
||||||
|
cells: Mapping[str, Mapping[str, Mapping[str, Union[int, float, str]]]] # preset -> project -> attribute
|
||||||
|
|
||||||
|
def __init__(self, diff: dict):
|
||||||
|
self.table_headers = sorted(self._find_all_preset_names(diff))
|
||||||
|
self.column_headers = sorted(self._find_all_attribute_names(diff))
|
||||||
|
self.row_headers = sorted(project for project in diff)
|
||||||
|
|
||||||
|
# All dimensions must have unique values
|
||||||
|
assert len(self.table_headers) == len(set(self.table_headers))
|
||||||
|
assert len(self.column_headers) == len(set(self.column_headers))
|
||||||
|
assert len(self.row_headers) == len(set(self.row_headers))
|
||||||
|
|
||||||
|
self.cells = {
|
||||||
|
preset: {
|
||||||
|
project: {
|
||||||
|
attribute: self._cell_content(diff, project, preset, attribute)
|
||||||
|
for attribute in self.column_headers
|
||||||
|
}
|
||||||
|
for project in self.row_headers
|
||||||
|
}
|
||||||
|
for preset in self.table_headers
|
||||||
|
}
|
||||||
|
|
||||||
|
def calculate_row_column_width(self) -> int:
|
||||||
|
return max(len(h) for h in self.row_headers)
|
||||||
|
|
||||||
|
def calculate_column_widths(self, table_header: str) -> Sequence[int]:
|
||||||
|
assert table_header in self.table_headers
|
||||||
|
|
||||||
|
return [
|
||||||
|
max(
|
||||||
|
len(column_header),
|
||||||
|
max(
|
||||||
|
len(str(self.cells[table_header][row_header][column_header]))
|
||||||
|
for row_header in self.row_headers
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for column_header in self.column_headers
|
||||||
|
]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _find_all_preset_names(cls, diff: dict) -> Set[str]:
|
||||||
|
return {
|
||||||
|
preset
|
||||||
|
for project, project_diff in diff.items()
|
||||||
|
if isinstance(project_diff, dict)
|
||||||
|
for preset in project_diff
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _find_all_attribute_names(cls, diff: dict) -> Set[str]:
|
||||||
|
return {
|
||||||
|
attribute
|
||||||
|
for project, project_diff in diff.items()
|
||||||
|
if isinstance(project_diff, dict)
|
||||||
|
for preset, preset_diff in project_diff.items()
|
||||||
|
if isinstance(preset_diff, dict)
|
||||||
|
for attribute in preset_diff
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _cell_content(cls, diff: dict, project: str, preset: str, attribute: str) -> str:
|
||||||
|
assert project in diff
|
||||||
|
|
||||||
|
if isinstance(diff[project], str):
|
||||||
|
return diff[project]
|
||||||
|
if preset not in diff[project]:
|
||||||
|
return ''
|
||||||
|
if isinstance(diff[project][preset], str):
|
||||||
|
return diff[project][preset]
|
||||||
|
if attribute not in diff[project][preset]:
|
||||||
|
return ''
|
||||||
|
|
||||||
|
return diff[project][preset][attribute]
|
||||||
|
|
||||||
|
|
||||||
|
class DiffTableFormatter:
|
||||||
|
LEGEND = dedent("""
|
||||||
|
`!V` = version mismatch
|
||||||
|
`!B` = no value in the "before" version
|
||||||
|
`!A` = no value in the "after" version
|
||||||
|
`!T` = one or both values were not numeric and could not be compared
|
||||||
|
`-0` = very small negative value rounded to zero
|
||||||
|
`+0` = very small positive value rounded to zero
|
||||||
|
""")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def run(cls, diff_table_set: DiffTableSet, output_format: OutputFormat):
|
||||||
|
if output_format == OutputFormat.JSON:
|
||||||
|
return json.dumps(diff_table_set.cells, indent=4, sort_keys=True)
|
||||||
|
else:
|
||||||
|
assert output_format in {OutputFormat.CONSOLE, OutputFormat.MARKDOWN}
|
||||||
|
|
||||||
|
output = ''
|
||||||
|
for table_header in diff_table_set.table_headers:
|
||||||
|
column_widths = ([
|
||||||
|
diff_table_set.calculate_row_column_width(),
|
||||||
|
*diff_table_set.calculate_column_widths(table_header)
|
||||||
|
])
|
||||||
|
|
||||||
|
if output_format == OutputFormat.MARKDOWN:
|
||||||
|
output += f'\n### `{table_header}`\n'
|
||||||
|
else:
|
||||||
|
output += f'\n{table_header.upper()}\n'
|
||||||
|
|
||||||
|
if output_format == OutputFormat.CONSOLE:
|
||||||
|
output += cls._format_separator_row(column_widths, output_format) + '\n'
|
||||||
|
output += cls._format_data_row(['project', *diff_table_set.column_headers], column_widths) + '\n'
|
||||||
|
output += cls._format_separator_row(column_widths, output_format) + '\n'
|
||||||
|
|
||||||
|
for row_header in diff_table_set.row_headers:
|
||||||
|
row = [
|
||||||
|
diff_table_set.cells[table_header][row_header][column_header]
|
||||||
|
for column_header in diff_table_set.column_headers
|
||||||
|
]
|
||||||
|
output += cls._format_data_row([row_header, *row], column_widths) + '\n'
|
||||||
|
|
||||||
|
if output_format == OutputFormat.CONSOLE:
|
||||||
|
output += cls._format_separator_row(column_widths, output_format) + '\n'
|
||||||
|
|
||||||
|
if output_format == OutputFormat.MARKDOWN:
|
||||||
|
output += f'\n{cls.LEGEND}\n'
|
||||||
|
return output
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _format_separator_row(cls, widths: Sequence[int], output_format: OutputFormat):
|
||||||
|
assert output_format in {OutputFormat.CONSOLE, OutputFormat.MARKDOWN}
|
||||||
|
|
||||||
|
if output_format == OutputFormat.MARKDOWN:
|
||||||
|
return '|:' + ':|-'.join('-' * width for width in widths) + ':|'
|
||||||
|
else:
|
||||||
|
return '|-' + '-|-'.join('-' * width for width in widths) + '-|'
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _format_data_row(cls, cells: Sequence[Union[int, float, str]], widths: Sequence[int]):
|
||||||
|
assert len(cells) == len(widths)
|
||||||
|
|
||||||
|
return '| ' + ' | '.join(str(cell).rjust(width) for cell, width in zip(cells, widths)) + ' |'
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class CommandLineOptions:
|
class CommandLineOptions:
|
||||||
|
diff_mode: DiffMode
|
||||||
report_before: Path
|
report_before: Path
|
||||||
report_after: Path
|
report_after: Path
|
||||||
difference_style: DifferenceStyle
|
difference_style: DifferenceStyle
|
||||||
relative_precision: int
|
relative_precision: int
|
||||||
|
output_format: OutputFormat
|
||||||
|
|
||||||
|
|
||||||
def process_commandline() -> CommandLineOptions:
|
def process_commandline() -> CommandLineOptions:
|
||||||
script_description = (
|
script_description = (
|
||||||
"Compares summarized benchmark reports and outputs JSON with the same structure but listing only differences."
|
"Compares summarized benchmark reports and outputs JSON with the same structure but listing only differences. "
|
||||||
|
"Can also print the output as markdown table and format the values to make differences stand out more."
|
||||||
)
|
)
|
||||||
|
|
||||||
parser = ArgumentParser(description=script_description)
|
parser = ArgumentParser(description=script_description)
|
||||||
|
parser.add_argument(
|
||||||
|
dest='diff_mode',
|
||||||
|
choices=[m.value for m in DiffMode],
|
||||||
|
help=(
|
||||||
|
"Diff mode: "
|
||||||
|
f"'{DiffMode.IN_PLACE.value}' preserves input JSON structure and replace values with differences; "
|
||||||
|
f"'{DiffMode.TABLE.value}' creates a table assuming 3-level project/preset/attribute structure."
|
||||||
|
)
|
||||||
|
)
|
||||||
parser.add_argument(dest='report_before', help="Path to a JSON file containing original benchmark results.")
|
parser.add_argument(dest='report_before', help="Path to a JSON file containing original benchmark results.")
|
||||||
parser.add_argument(dest='report_after', help="Path to a JSON file containing new benchmark results.")
|
parser.add_argument(dest='report_after', help="Path to a JSON file containing new benchmark results.")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@ -156,7 +366,8 @@ def process_commandline() -> CommandLineOptions:
|
|||||||
f"'{DifferenceStyle.RELATIVE.value}' also divides by the original; "
|
f"'{DifferenceStyle.RELATIVE.value}' also divides by the original; "
|
||||||
f"'{DifferenceStyle.HUMANIZED.value}' is like relative but value is a percentage and "
|
f"'{DifferenceStyle.HUMANIZED.value}' is like relative but value is a percentage and "
|
||||||
"positive/negative changes are emphasized. "
|
"positive/negative changes are emphasized. "
|
||||||
f"(default: '{DEFAULT_DIFFERENCE_STYLE}')."
|
f"(default: '{DEFAULT_DIFFERENCE_STYLE[DiffMode.IN_PLACE]}' in '{DiffMode.IN_PLACE.value}' mode, "
|
||||||
|
f"'{DEFAULT_DIFFERENCE_STYLE[DiffMode.TABLE]}' in '{DiffMode.TABLE.value}' mode)"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
# NOTE: Negative values are valid for precision. round() handles them in a sensible way.
|
# NOTE: Negative values are valid for precision. round() handles them in a sensible way.
|
||||||
@ -173,21 +384,47 @@ def process_commandline() -> CommandLineOptions:
|
|||||||
f"(default: {DEFAULT_RELATIVE_PRECISION})"
|
f"(default: {DEFAULT_RELATIVE_PRECISION})"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--output-format',
|
||||||
|
dest='output_format',
|
||||||
|
choices=[o.value for o in OutputFormat],
|
||||||
|
help=(
|
||||||
|
"The format to use for the diff: "
|
||||||
|
f"'{OutputFormat.JSON.value}' is raw JSON; "
|
||||||
|
f"'{OutputFormat.CONSOLE.value}' is a table with human-readable values that will look good in the console output. "
|
||||||
|
f"'{OutputFormat.MARKDOWN.value}' is similar '{OutputFormat.CONSOLE.value}' but adjusted to "
|
||||||
|
"render as proper markdown and with extra elements (legend, emoji to make non-zero values stand out more, etc)."
|
||||||
|
f"(default: '{DEFAULT_OUTPUT_FORMAT[DiffMode.IN_PLACE]}' in '{DiffMode.IN_PLACE.value}' mode, "
|
||||||
|
f"'{DEFAULT_OUTPUT_FORMAT[DiffMode.TABLE]}' in '{DiffMode.TABLE.value}' mode)"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
options = parser.parse_args()
|
options = parser.parse_args()
|
||||||
|
|
||||||
if options.difference_style is not None:
|
if options.difference_style is not None:
|
||||||
difference_style = DifferenceStyle(options.difference_style)
|
difference_style = DifferenceStyle(options.difference_style)
|
||||||
else:
|
else:
|
||||||
difference_style = DEFAULT_DIFFERENCE_STYLE
|
difference_style = DEFAULT_DIFFERENCE_STYLE[DiffMode(options.diff_mode)]
|
||||||
|
|
||||||
|
if options.output_format is not None:
|
||||||
|
output_format = OutputFormat(options.output_format)
|
||||||
|
else:
|
||||||
|
output_format = DEFAULT_OUTPUT_FORMAT[DiffMode(options.diff_mode)]
|
||||||
|
|
||||||
processed_options = CommandLineOptions(
|
processed_options = CommandLineOptions(
|
||||||
|
diff_mode=DiffMode(options.diff_mode),
|
||||||
report_before=Path(options.report_before),
|
report_before=Path(options.report_before),
|
||||||
report_after=Path(options.report_after),
|
report_after=Path(options.report_after),
|
||||||
difference_style=difference_style,
|
difference_style=difference_style,
|
||||||
relative_precision=options.relative_precision,
|
relative_precision=options.relative_precision,
|
||||||
|
output_format=output_format,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if processed_options.diff_mode == DiffMode.IN_PLACE and processed_options.output_format != OutputFormat.JSON:
|
||||||
|
raise CommandLineError(
|
||||||
|
f"Only the '{OutputFormat.JSON.value}' output format is supported in the '{DiffMode.IN_PLACE.value}' mode."
|
||||||
|
)
|
||||||
|
|
||||||
return processed_options
|
return processed_options
|
||||||
|
|
||||||
|
|
||||||
@ -195,13 +432,17 @@ def main():
|
|||||||
try:
|
try:
|
||||||
options = process_commandline()
|
options = process_commandline()
|
||||||
|
|
||||||
differ = BenchmarkDiffer(options.difference_style, options.relative_precision)
|
differ = BenchmarkDiffer(options.difference_style, options.relative_precision, options.output_format)
|
||||||
diff = differ.run(
|
diff = differ.run(
|
||||||
json.loads(options.report_before.read_text('utf-8')),
|
json.loads(options.report_before.read_text('utf-8')),
|
||||||
json.loads(options.report_after.read_text('utf-8')),
|
json.loads(options.report_after.read_text('utf-8')),
|
||||||
)
|
)
|
||||||
|
|
||||||
print(json.dumps(diff, indent=4, sort_keys=True))
|
if options.diff_mode == DiffMode.IN_PLACE:
|
||||||
|
print(json.dumps(diff, indent=4, sort_keys=True))
|
||||||
|
else:
|
||||||
|
assert options.diff_mode == DiffMode.TABLE
|
||||||
|
print(DiffTableFormatter.run(DiffTableSet(diff), options.output_format))
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
except CommandLineError as exception:
|
except CommandLineError as exception:
|
||||||
|
@ -0,0 +1,75 @@
|
|||||||
|
|
||||||
|
### `ir-no-optimize`
|
||||||
|
| project | bytecode_size | deployment_gas | method_gas |
|
||||||
|
|:---------:|---------------:|---------------:|---------------:|
|
||||||
|
| bleeps | | | |
|
||||||
|
| colony | | | |
|
||||||
|
| elementfi | | | `0%` |
|
||||||
|
| ens | `!A` | `!A` | `!A` |
|
||||||
|
| euler | **`+1.43% ❌`** | `0%` | **`+2.47% ❌`** |
|
||||||
|
| gnosis | `!B` | `!B` | `!B` |
|
||||||
|
| zeppelin | | | |
|
||||||
|
|
||||||
|
### `ir-optimize-evm+yul`
|
||||||
|
| project | bytecode_size | deployment_gas | method_gas |
|
||||||
|
|:---------:|----------------:|----------------:|-----------:|
|
||||||
|
| bleeps | **`+0.53% ❌`** | `0%` | `-0%` |
|
||||||
|
| colony | `!A` | `!A` | `!A` |
|
||||||
|
| elementfi | | | |
|
||||||
|
| ens | `!A` | `!A` | `!A` |
|
||||||
|
| euler | **`+12.64% ❌`** | **`+11.98% ❌`** | `0%` |
|
||||||
|
| gnosis | `!B` | `!B` | `!B` |
|
||||||
|
| zeppelin | | | |
|
||||||
|
|
||||||
|
### `ir-optimize-evm-only`
|
||||||
|
| project | bytecode_size | deployment_gas | method_gas |
|
||||||
|
|:---------:|--------------:|---------------:|-----------:|
|
||||||
|
| bleeps | | | |
|
||||||
|
| colony | | | |
|
||||||
|
| elementfi | `!B` | `!B` | `!B` |
|
||||||
|
| ens | `!A` | `!A` | `!A` |
|
||||||
|
| euler | `!V` | `!V` | `!V` |
|
||||||
|
| gnosis | `!B` | `!B` | `!B` |
|
||||||
|
| zeppelin | | | |
|
||||||
|
|
||||||
|
### `legacy-no-optimize`
|
||||||
|
| project | bytecode_size | deployment_gas | method_gas |
|
||||||
|
|:---------:|--------------:|---------------:|-----------:|
|
||||||
|
| bleeps | | | |
|
||||||
|
| colony | `!B` | `!B` | `!B` |
|
||||||
|
| elementfi | `!A` | `!B` | |
|
||||||
|
| ens | `!A` | `!A` | `!A` |
|
||||||
|
| euler | `!V` | `!V` | `!V` |
|
||||||
|
| gnosis | `!B` | `!B` | `!B` |
|
||||||
|
| zeppelin | | | |
|
||||||
|
|
||||||
|
### `legacy-optimize-evm+yul`
|
||||||
|
| project | bytecode_size | deployment_gas | method_gas |
|
||||||
|
|:---------:|--------------:|---------------:|-----------:|
|
||||||
|
| bleeps | `0%` | `0%` | `0%` |
|
||||||
|
| colony | `0%` | | |
|
||||||
|
| elementfi | `!A` | `!B` | |
|
||||||
|
| ens | `!A` | `!A` | `!A` |
|
||||||
|
| euler | `!V` | `!V` | `!V` |
|
||||||
|
| gnosis | `!B` | `!B` | `!B` |
|
||||||
|
| zeppelin | `0%` | `0%` | |
|
||||||
|
|
||||||
|
### `legacy-optimize-evm-only`
|
||||||
|
| project | bytecode_size | deployment_gas | method_gas |
|
||||||
|
|:---------:|--------------:|---------------:|-----------:|
|
||||||
|
| bleeps | | | |
|
||||||
|
| colony | | | |
|
||||||
|
| elementfi | `!A` | `!A` | `!A` |
|
||||||
|
| ens | `!A` | `!A` | `!A` |
|
||||||
|
| euler | `!V` | `!V` | `!V` |
|
||||||
|
| gnosis | `!B` | `!B` | `!B` |
|
||||||
|
| zeppelin | | | |
|
||||||
|
|
||||||
|
|
||||||
|
`!V` = version mismatch
|
||||||
|
`!B` = no value in the "before" version
|
||||||
|
`!A` = no value in the "after" version
|
||||||
|
`!T` = one or both values were not numeric and could not be compared
|
||||||
|
`-0` = very small negative value rounded to zero
|
||||||
|
`+0` = very small positive value rounded to zero
|
||||||
|
|
@ -1,5 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from textwrap import dedent
|
||||||
import json
|
import json
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
@ -7,12 +8,15 @@ from unittest_helpers import FIXTURE_DIR, load_fixture
|
|||||||
|
|
||||||
# NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
|
# NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
|
||||||
# pragma pylint: disable=import-error
|
# pragma pylint: disable=import-error
|
||||||
from externalTests.benchmark_diff import BenchmarkDiffer, DifferenceStyle
|
from externalTests.benchmark_diff import BenchmarkDiffer, DifferenceStyle, DiffTableSet, DiffTableFormatter, OutputFormat
|
||||||
# pragma pylint: enable=import-error
|
# pragma pylint: enable=import-error
|
||||||
|
|
||||||
SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-develop.json'
|
SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-develop.json'
|
||||||
SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-branch.json'
|
SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-branch.json'
|
||||||
|
|
||||||
|
SUMMARIZED_DIFF_HUMANIZED_MD_PATH = FIXTURE_DIR / 'summarized-benchmark-diff-develop-branch-humanized.md'
|
||||||
|
SUMMARIZED_DIFF_HUMANIZED_MD = load_fixture(SUMMARIZED_DIFF_HUMANIZED_MD_PATH)
|
||||||
|
|
||||||
|
|
||||||
class TestBenchmarkDiff(unittest.TestCase):
|
class TestBenchmarkDiff(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
@ -108,7 +112,7 @@ class TestBenchmarkDiff(unittest.TestCase):
|
|||||||
"gnosis": "!B",
|
"gnosis": "!B",
|
||||||
"ens": "!A",
|
"ens": "!A",
|
||||||
}
|
}
|
||||||
differ = BenchmarkDiffer(DifferenceStyle.ABSOLUTE, None)
|
differ = BenchmarkDiffer(DifferenceStyle.ABSOLUTE, None, OutputFormat.JSON)
|
||||||
self.assertEqual(differ.run(report_before, report_after), expected_diff)
|
self.assertEqual(differ.run(report_before, report_after), expected_diff)
|
||||||
|
|
||||||
|
|
||||||
@ -138,105 +142,137 @@ class TestBenchmarkDiffer(unittest.TestCase):
|
|||||||
|
|
||||||
def test_empty(self):
|
def test_empty(self):
|
||||||
for style in DifferenceStyle:
|
for style in DifferenceStyle:
|
||||||
differ = BenchmarkDiffer(style, None)
|
differ = BenchmarkDiffer(style, None, OutputFormat.JSON)
|
||||||
self._assert_single_value_diff_matches(differ, [({}, {}, {})], nest_result=False)
|
self._assert_single_value_diff_matches(differ, [({}, {}, {})], nest_result=False)
|
||||||
|
|
||||||
def test_null(self):
|
def test_null(self):
|
||||||
for style in DifferenceStyle:
|
for style in DifferenceStyle:
|
||||||
differ = BenchmarkDiffer(style, None)
|
differ = BenchmarkDiffer(style, None, OutputFormat.JSON)
|
||||||
self._assert_single_value_diff_matches(differ, [(None, None, {})], nest_result=False)
|
self._assert_single_value_diff_matches(differ, [(None, None, {})], nest_result=False)
|
||||||
|
|
||||||
def test_number_diff_absolute_json(self):
|
def test_number_diff_absolute_json(self):
|
||||||
self._assert_single_value_diff_matches(
|
for output_format in OutputFormat:
|
||||||
BenchmarkDiffer(DifferenceStyle.ABSOLUTE, 4),
|
self._assert_single_value_diff_matches(
|
||||||
[
|
BenchmarkDiffer(DifferenceStyle.ABSOLUTE, 4, output_format),
|
||||||
(2, 2, 0),
|
[
|
||||||
(2, 5, 3),
|
(2, 2, 0),
|
||||||
(5, 2, -3),
|
(2, 5, 3),
|
||||||
(2.0, 2.0, 0),
|
(5, 2, -3),
|
||||||
(2, 2.0, 0),
|
(2.0, 2.0, 0),
|
||||||
(2.0, 2, 0),
|
(2, 2.0, 0),
|
||||||
(2, 2.5, 2.5 - 2),
|
(2.0, 2, 0),
|
||||||
(2.5, 2, 2 - 2.5),
|
(2, 2.5, 2.5 - 2),
|
||||||
|
(2.5, 2, 2 - 2.5),
|
||||||
|
|
||||||
(0, 0, 0),
|
(0, 0, 0),
|
||||||
(0, 2, 2),
|
(0, 2, 2),
|
||||||
(0, -2, -2),
|
(0, -2, -2),
|
||||||
|
|
||||||
(-3, -1, 2),
|
(-3, -1, 2),
|
||||||
(-1, -3, -2),
|
(-1, -3, -2),
|
||||||
(2, 0, -2),
|
(2, 0, -2),
|
||||||
(-2, 0, 2),
|
(-2, 0, 2),
|
||||||
|
|
||||||
(1.00006, 1, 1 - 1.00006),
|
(1.00006, 1, 1 - 1.00006),
|
||||||
(1, 1.00006, 1.00006 - 1),
|
(1, 1.00006, 1.00006 - 1),
|
||||||
(1.00004, 1, 1 - 1.00004),
|
(1.00004, 1, 1 - 1.00004),
|
||||||
(1, 1.00004, 1.00004 - 1),
|
(1, 1.00004, 1.00004 - 1),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_number_diff_json(self):
|
def test_number_diff_json(self):
|
||||||
|
for output_format in OutputFormat:
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(DifferenceStyle.RELATIVE, 4, output_format),
|
||||||
|
[
|
||||||
|
(2, 2, 0),
|
||||||
|
(2, 5, (5 - 2) / 2),
|
||||||
|
(5, 2, (2 - 5) / 5),
|
||||||
|
(2.0, 2.0, 0),
|
||||||
|
(2, 2.0, 0),
|
||||||
|
(2.0, 2, 0),
|
||||||
|
(2, 2.5, (2.5 - 2) / 2),
|
||||||
|
(2.5, 2, (2 - 2.5) / 2.5),
|
||||||
|
|
||||||
|
(0, 0, 0),
|
||||||
|
(0, 2, '+INF'),
|
||||||
|
(0, -2, '-INF'),
|
||||||
|
|
||||||
|
(-3, -1, 0.6667),
|
||||||
|
(-1, -3, -2),
|
||||||
|
(2, 0, -1),
|
||||||
|
(-2, 0, 1),
|
||||||
|
|
||||||
|
(1.00006, 1, -0.0001),
|
||||||
|
(1, 1.00006, 0.0001),
|
||||||
|
(1.000004, 1, '-0'),
|
||||||
|
(1, 1.000004, '+0'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_number_diff_humanized_json_and_console(self):
|
||||||
|
for output_format in [OutputFormat.JSON, OutputFormat.CONSOLE]:
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, output_format),
|
||||||
|
[
|
||||||
|
(2, 2, '0%'),
|
||||||
|
(2, 5, '+150%'),
|
||||||
|
(5, 2, '-60%'),
|
||||||
|
(2.0, 2.0, '0%'),
|
||||||
|
(2, 2.0, '0%'),
|
||||||
|
(2.0, 2, '0%'),
|
||||||
|
(2, 2.5, '+25%'),
|
||||||
|
(2.5, 2, '-20%'),
|
||||||
|
|
||||||
|
(0, 0, '0%'),
|
||||||
|
(0, 2, '+INF%'),
|
||||||
|
(0, -2, '-INF%'),
|
||||||
|
|
||||||
|
(-3, -1, '+66.67%'),
|
||||||
|
(-1, -3, '-200%'),
|
||||||
|
(2, 0, '-100%'),
|
||||||
|
(-2, 0, '+100%'),
|
||||||
|
|
||||||
|
(1.00006, 1, '-0.01%'),
|
||||||
|
(1, 1.00006, '+0.01%'),
|
||||||
|
(1.000004, 1, '-0%'),
|
||||||
|
(1, 1.000004, '+0%'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_number_diff_humanized_markdown(self):
|
||||||
self._assert_single_value_diff_matches(
|
self._assert_single_value_diff_matches(
|
||||||
BenchmarkDiffer(DifferenceStyle.RELATIVE, 4),
|
BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, OutputFormat.MARKDOWN),
|
||||||
[
|
[
|
||||||
(2, 2, 0),
|
(2, 2, '`0%`'),
|
||||||
(2, 5, (5 - 2) / 2),
|
(2, 5, '**`+150% ❌`**'),
|
||||||
(5, 2, (2 - 5) / 5),
|
(5, 2, '**`-60% ✅`**'),
|
||||||
(2.0, 2.0, 0),
|
(2.0, 2.0, '`0%`'),
|
||||||
(2, 2.0, 0),
|
(2, 2.0, '`0%`'),
|
||||||
(2.0, 2, 0),
|
(2.0, 2, '`0%`'),
|
||||||
(2, 2.5, (2.5 - 2) / 2),
|
(2, 2.5, '**`+25% ❌`**'),
|
||||||
(2.5, 2, (2 - 2.5) / 2.5),
|
(2.5, 2, '**`-20% ✅`**'),
|
||||||
|
|
||||||
(0, 0, 0),
|
(0, 0, '`0%`'),
|
||||||
(0, 2, '+INF'),
|
(0, 2, '`+INF%`'),
|
||||||
(0, -2, '-INF'),
|
(0, -2, '`-INF%`'),
|
||||||
|
|
||||||
(-3, -1, 0.6667),
|
(-3, -1, '**`+66.67% ❌`**'),
|
||||||
(-1, -3, -2),
|
(-1, -3, '**`-200% ✅`**'),
|
||||||
(2, 0, -1),
|
(2, 0, '**`-100% ✅`**'),
|
||||||
(-2, 0, 1),
|
(-2, 0, '**`+100% ❌`**'),
|
||||||
|
|
||||||
(1.00006, 1, -0.0001),
|
(1.00006, 1, '**`-0.01% ✅`**'),
|
||||||
(1, 1.00006, 0.0001),
|
(1, 1.00006, '**`+0.01% ❌`**'),
|
||||||
(1.000004, 1, '-0'),
|
(1.000004, 1, '`-0%`'),
|
||||||
(1, 1.000004, '+0'),
|
(1, 1.000004, '`+0%`'),
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_number_diff_humanized_json(self):
|
|
||||||
self._assert_single_value_diff_matches(
|
|
||||||
BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4),
|
|
||||||
[
|
|
||||||
(2, 2, '0%'),
|
|
||||||
(2, 5, '+150%'),
|
|
||||||
(5, 2, '-60%'),
|
|
||||||
(2.0, 2.0, '0%'),
|
|
||||||
(2, 2.0, '0%'),
|
|
||||||
(2.0, 2, '0%'),
|
|
||||||
(2, 2.5, '+25%'),
|
|
||||||
(2.5, 2, '-20%'),
|
|
||||||
|
|
||||||
(0, 0, '0%'),
|
|
||||||
(0, 2, '+INF%'),
|
|
||||||
(0, -2, '-INF%'),
|
|
||||||
|
|
||||||
(-3, -1, '+66.67%'),
|
|
||||||
(-1, -3, '-200%'),
|
|
||||||
(2, 0, '-100%'),
|
|
||||||
(-2, 0, '+100%'),
|
|
||||||
|
|
||||||
(1.00006, 1, '-0.01%'),
|
|
||||||
(1, 1.00006, '+0.01%'),
|
|
||||||
(1.000004, 1, '-0%'),
|
|
||||||
(1, 1.000004, '+0%'),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_type_mismatch(self):
|
def test_type_mismatch(self):
|
||||||
for style in DifferenceStyle:
|
for style in DifferenceStyle:
|
||||||
self._assert_single_value_diff_matches(
|
self._assert_single_value_diff_matches(
|
||||||
BenchmarkDiffer(style, 4),
|
BenchmarkDiffer(style, 4, OutputFormat.JSON),
|
||||||
[
|
[
|
||||||
(1, {}, '!T'),
|
(1, {}, '!T'),
|
||||||
({}, 1, '!T'),
|
({}, 1, '!T'),
|
||||||
@ -255,7 +291,7 @@ class TestBenchmarkDiffer(unittest.TestCase):
|
|||||||
def test_version_mismatch(self):
|
def test_version_mismatch(self):
|
||||||
for style in DifferenceStyle:
|
for style in DifferenceStyle:
|
||||||
self._assert_single_value_diff_matches(
|
self._assert_single_value_diff_matches(
|
||||||
BenchmarkDiffer(style, 4),
|
BenchmarkDiffer(style, 4, OutputFormat.JSON),
|
||||||
[
|
[
|
||||||
({'a': 123, 'version': 1}, {'a': 123, 'version': 2}, '!V'),
|
({'a': 123, 'version': 1}, {'a': 123, 'version': 2}, '!V'),
|
||||||
({'a': 123, 'version': 2}, {'a': 123, 'version': 1}, '!V'),
|
({'a': 123, 'version': 2}, {'a': 123, 'version': 1}, '!V'),
|
||||||
@ -275,7 +311,7 @@ class TestBenchmarkDiffer(unittest.TestCase):
|
|||||||
def test_missing(self):
|
def test_missing(self):
|
||||||
for style in DifferenceStyle:
|
for style in DifferenceStyle:
|
||||||
self._assert_single_value_diff_matches(
|
self._assert_single_value_diff_matches(
|
||||||
BenchmarkDiffer(style, None),
|
BenchmarkDiffer(style, None, OutputFormat.JSON),
|
||||||
[
|
[
|
||||||
(1, None, '!A'),
|
(1, None, '!A'),
|
||||||
(None, 1, '!B'),
|
(None, 1, '!B'),
|
||||||
@ -300,10 +336,173 @@ class TestBenchmarkDiffer(unittest.TestCase):
|
|||||||
def test_missing_vs_null(self):
|
def test_missing_vs_null(self):
|
||||||
for style in DifferenceStyle:
|
for style in DifferenceStyle:
|
||||||
self._assert_single_value_diff_matches(
|
self._assert_single_value_diff_matches(
|
||||||
BenchmarkDiffer(style, None),
|
BenchmarkDiffer(style, None, OutputFormat.JSON),
|
||||||
[
|
[
|
||||||
({'a': None}, {}, {}),
|
({'a': None}, {}, {}),
|
||||||
({}, {'a': None}, {}),
|
({}, {'a': None}, {}),
|
||||||
],
|
],
|
||||||
nest_result=False,
|
nest_result=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDiffTableFormatter(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.maxDiff = 10000
|
||||||
|
|
||||||
|
self.report_before = {
|
||||||
|
'project A': {
|
||||||
|
'preset X': {'A1': 99, 'A2': 50, 'version': 1},
|
||||||
|
'preset Y': {'A1': 0, 'A2': 50, 'version': 1},
|
||||||
|
},
|
||||||
|
'project B': {
|
||||||
|
'preset X': { 'A2': 50},
|
||||||
|
'preset Y': {'A1': 0},
|
||||||
|
},
|
||||||
|
'project C': {
|
||||||
|
'preset X': {'A1': 0, 'A2': 50, 'version': 1},
|
||||||
|
},
|
||||||
|
'project D': {
|
||||||
|
'preset X': {'A1': 999},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
self.report_after = {
|
||||||
|
'project A': {
|
||||||
|
'preset X': {'A1': 100, 'A2': 50, 'version': 1},
|
||||||
|
'preset Y': {'A1': 500, 'A2': 500, 'version': 2},
|
||||||
|
},
|
||||||
|
'project B': {
|
||||||
|
'preset X': {'A1': 0},
|
||||||
|
'preset Y': { 'A2': 50},
|
||||||
|
},
|
||||||
|
'project C': {
|
||||||
|
'preset Y': {'A1': 0, 'A2': 50, 'version': 1},
|
||||||
|
},
|
||||||
|
'project E': {
|
||||||
|
'preset Y': { 'A2': 999},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_diff_table_formatter(self):
|
||||||
|
report_before = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH))
|
||||||
|
report_after = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH))
|
||||||
|
differ = BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, OutputFormat.MARKDOWN)
|
||||||
|
diff = differ.run(report_before, report_after)
|
||||||
|
|
||||||
|
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.MARKDOWN), SUMMARIZED_DIFF_HUMANIZED_MD)
|
||||||
|
|
||||||
|
def test_diff_table_formatter_json_absolute(self):
|
||||||
|
differ = BenchmarkDiffer(DifferenceStyle.ABSOLUTE, 4, OutputFormat.JSON)
|
||||||
|
diff = differ.run(self.report_before, self.report_after)
|
||||||
|
|
||||||
|
expected_formatted_table = dedent("""\
|
||||||
|
{
|
||||||
|
"preset X": {
|
||||||
|
"project A": {
|
||||||
|
"A1": 1,
|
||||||
|
"A2": 0
|
||||||
|
},
|
||||||
|
"project B": {
|
||||||
|
"A1": "!B",
|
||||||
|
"A2": "!A"
|
||||||
|
},
|
||||||
|
"project C": {
|
||||||
|
"A1": "!A",
|
||||||
|
"A2": "!A"
|
||||||
|
},
|
||||||
|
"project D": {
|
||||||
|
"A1": "!A",
|
||||||
|
"A2": "!A"
|
||||||
|
},
|
||||||
|
"project E": {
|
||||||
|
"A1": "!B",
|
||||||
|
"A2": "!B"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"preset Y": {
|
||||||
|
"project A": {
|
||||||
|
"A1": "!V",
|
||||||
|
"A2": "!V"
|
||||||
|
},
|
||||||
|
"project B": {
|
||||||
|
"A1": "!A",
|
||||||
|
"A2": "!B"
|
||||||
|
},
|
||||||
|
"project C": {
|
||||||
|
"A1": "!B",
|
||||||
|
"A2": "!B"
|
||||||
|
},
|
||||||
|
"project D": {
|
||||||
|
"A1": "!A",
|
||||||
|
"A2": "!A"
|
||||||
|
},
|
||||||
|
"project E": {
|
||||||
|
"A1": "!B",
|
||||||
|
"A2": "!B"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.JSON), expected_formatted_table)
|
||||||
|
|
||||||
|
def test_diff_table_formatter_console_relative(self):
|
||||||
|
differ = BenchmarkDiffer(DifferenceStyle.RELATIVE, 4, OutputFormat.CONSOLE)
|
||||||
|
diff = differ.run(self.report_before, self.report_after)
|
||||||
|
|
||||||
|
expected_formatted_table = dedent("""
|
||||||
|
PRESET X
|
||||||
|
|-----------|--------|----|
|
||||||
|
| project | A1 | A2 |
|
||||||
|
|-----------|--------|----|
|
||||||
|
| project A | 0.0101 | 0 |
|
||||||
|
| project B | !B | !A |
|
||||||
|
| project C | !A | !A |
|
||||||
|
| project D | !A | !A |
|
||||||
|
| project E | !B | !B |
|
||||||
|
|-----------|--------|----|
|
||||||
|
|
||||||
|
PRESET Y
|
||||||
|
|-----------|----|----|
|
||||||
|
| project | A1 | A2 |
|
||||||
|
|-----------|----|----|
|
||||||
|
| project A | !V | !V |
|
||||||
|
| project B | !A | !B |
|
||||||
|
| project C | !B | !B |
|
||||||
|
| project D | !A | !A |
|
||||||
|
| project E | !B | !B |
|
||||||
|
|-----------|----|----|
|
||||||
|
""")
|
||||||
|
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.CONSOLE), expected_formatted_table)
|
||||||
|
|
||||||
|
def test_diff_table_formatter_markdown_humanized(self):
|
||||||
|
differ = BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, OutputFormat.MARKDOWN)
|
||||||
|
diff = differ.run(self.report_before, self.report_after)
|
||||||
|
|
||||||
|
expected_formatted_table = dedent("""
|
||||||
|
### `preset X`
|
||||||
|
| project | A1 | A2 |
|
||||||
|
|:---------:|---------------:|-----:|
|
||||||
|
| project A | **`+1.01% ❌`** | `0%` |
|
||||||
|
| project B | `!B` | `!A` |
|
||||||
|
| project C | `!A` | `!A` |
|
||||||
|
| project D | `!A` | `!A` |
|
||||||
|
| project E | `!B` | `!B` |
|
||||||
|
|
||||||
|
### `preset Y`
|
||||||
|
| project | A1 | A2 |
|
||||||
|
|:---------:|-----:|-----:|
|
||||||
|
| project A | `!V` | `!V` |
|
||||||
|
| project B | `!A` | `!B` |
|
||||||
|
| project C | `!B` | `!B` |
|
||||||
|
| project D | `!A` | `!A` |
|
||||||
|
| project E | `!B` | `!B` |
|
||||||
|
|
||||||
|
|
||||||
|
`!V` = version mismatch
|
||||||
|
`!B` = no value in the "before" version
|
||||||
|
`!A` = no value in the "after" version
|
||||||
|
`!T` = one or both values were not numeric and could not be compared
|
||||||
|
`-0` = very small negative value rounded to zero
|
||||||
|
`+0` = very small positive value rounded to zero
|
||||||
|
|
||||||
|
""")
|
||||||
|
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.MARKDOWN), expected_formatted_table)
|
||||||
|
Loading…
Reference in New Issue
Block a user