mirror of
https://github.com/ethereum/solidity
synced 2023-10-03 13:03:40 +00:00
Merge pull request #12804 from ethereum/benchmark-differ
Benchmark differ
This commit is contained in:
Daniel Kirchner
@@ -0,0 +1,75 @@
|
||||
|
||||
### `ir-no-optimize`
|
||||
| project | bytecode_size | deployment_gas | method_gas |
|
||||
|:---------:|---------------:|---------------:|---------------:|
|
||||
| bleeps | | | |
|
||||
| colony | | | |
|
||||
| elementfi | | | `0%` |
|
||||
| ens | `!A` | `!A` | `!A` |
|
||||
| euler | **`+1.43% ❌`** | `0%` | **`+2.47% ❌`** |
|
||||
| gnosis | `!B` | `!B` | `!B` |
|
||||
| zeppelin | | | |
|
||||
|
||||
### `ir-optimize-evm+yul`
|
||||
| project | bytecode_size | deployment_gas | method_gas |
|
||||
|:---------:|----------------:|----------------:|-----------:|
|
||||
| bleeps | **`+0.53% ❌`** | `0%` | `-0%` |
|
||||
| colony | `!A` | `!A` | `!A` |
|
||||
| elementfi | | | |
|
||||
| ens | `!A` | `!A` | `!A` |
|
||||
| euler | **`+12.64% ❌`** | **`+11.98% ❌`** | `0%` |
|
||||
| gnosis | `!B` | `!B` | `!B` |
|
||||
| zeppelin | | | |
|
||||
|
||||
### `ir-optimize-evm-only`
|
||||
| project | bytecode_size | deployment_gas | method_gas |
|
||||
|:---------:|--------------:|---------------:|-----------:|
|
||||
| bleeps | | | |
|
||||
| colony | | | |
|
||||
| elementfi | `!B` | `!B` | `!B` |
|
||||
| ens | `!A` | `!A` | `!A` |
|
||||
| euler | `!V` | `!V` | `!V` |
|
||||
| gnosis | `!B` | `!B` | `!B` |
|
||||
| zeppelin | | | |
|
||||
|
||||
### `legacy-no-optimize`
|
||||
| project | bytecode_size | deployment_gas | method_gas |
|
||||
|:---------:|--------------:|---------------:|-----------:|
|
||||
| bleeps | | | |
|
||||
| colony | `!B` | `!B` | `!B` |
|
||||
| elementfi | `!A` | `!B` | |
|
||||
| ens | `!A` | `!A` | `!A` |
|
||||
| euler | `!V` | `!V` | `!V` |
|
||||
| gnosis | `!B` | `!B` | `!B` |
|
||||
| zeppelin | | | |
|
||||
|
||||
### `legacy-optimize-evm+yul`
|
||||
| project | bytecode_size | deployment_gas | method_gas |
|
||||
|:---------:|--------------:|---------------:|-----------:|
|
||||
| bleeps | `0%` | `0%` | `0%` |
|
||||
| colony | `0%` | | |
|
||||
| elementfi | `!A` | `!B` | |
|
||||
| ens | `!A` | `!A` | `!A` |
|
||||
| euler | `!V` | `!V` | `!V` |
|
||||
| gnosis | `!B` | `!B` | `!B` |
|
||||
| zeppelin | `0%` | `0%` | |
|
||||
|
||||
### `legacy-optimize-evm-only`
|
||||
| project | bytecode_size | deployment_gas | method_gas |
|
||||
|:---------:|--------------:|---------------:|-----------:|
|
||||
| bleeps | | | |
|
||||
| colony | | | |
|
||||
| elementfi | `!A` | `!A` | `!A` |
|
||||
| ens | `!A` | `!A` | `!A` |
|
||||
| euler | `!V` | `!V` | `!V` |
|
||||
| gnosis | `!B` | `!B` | `!B` |
|
||||
| zeppelin | | | |
|
||||
|
||||
|
||||
`!V` = version mismatch
|
||||
`!B` = no value in the "before" version
|
||||
`!A` = no value in the "after" version
|
||||
`!T` = one or both values were not numeric and could not be compared
|
||||
`-0` = very small negative value rounded to zero
|
||||
`+0` = very small positive value rounded to zero
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"bleeps": {
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 132868,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 39289198,
|
||||
"version": "bb90cd0"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 137869,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 38863224,
|
||||
"version": "bb90cd0"
|
||||
}
|
||||
},
|
||||
"colony": {
|
||||
"legacy-no-optimize": {
|
||||
"bytecode_size": 664190,
|
||||
"deployment_gas": null,
|
||||
"method_gas": null,
|
||||
"version": "573399b"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 363606,
|
||||
"deployment_gas": null,
|
||||
"method_gas": null,
|
||||
"version": "573399b"
|
||||
}
|
||||
},
|
||||
"elementfi": {
|
||||
"legacy-no-optimize": {
|
||||
"bytecode_size": null,
|
||||
"deployment_gas": 69200158,
|
||||
"method_gas": null,
|
||||
"version": "87f8b5e"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"deployment_gas": 40951128,
|
||||
"version": "87f8b5e"
|
||||
},
|
||||
"ir-optimize-evm-only": {},
|
||||
"ir-no-optimize": {
|
||||
"deployment_gas": null,
|
||||
"method_gas": 2777867251,
|
||||
"version": "87f8b5e"
|
||||
}
|
||||
},
|
||||
"euler": {
|
||||
"ir-no-optimize": {
|
||||
"bytecode_size": 328540,
|
||||
"deployment_gas": 61591870,
|
||||
"method_gas": 3537419168,
|
||||
"version": "2ef99fc"
|
||||
},
|
||||
"legacy-no-optimize": {
|
||||
"bytecode_size": 328540,
|
||||
"deployment_gas": 62590688,
|
||||
"method_gas": 3537419168,
|
||||
"version": "2ef99fc"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 182190,
|
||||
"deployment_gas": 35236828,
|
||||
"method_gas": 2777867251,
|
||||
"version": "2ef99fc"
|
||||
},
|
||||
"legacy-optimize-evm-only": {
|
||||
"bytecode_size": 205211,
|
||||
"deployment_gas": 39459629,
|
||||
"method_gas": 2978467272,
|
||||
"version": "2ef99fc"
|
||||
},
|
||||
"ir-optimize-evm-only": {
|
||||
"bytecode_size": 205211,
|
||||
"deployment_gas": 39459629,
|
||||
"method_gas": 2978467272,
|
||||
"version": "2ef99fc"
|
||||
},
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 205211,
|
||||
"deployment_gas": 39459629,
|
||||
"method_gas": 2777867251
|
||||
}
|
||||
},
|
||||
"gnosis": {
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 56069,
|
||||
"deployment_gas": null,
|
||||
"method_gas": null,
|
||||
"version": "ea09294"
|
||||
}
|
||||
},
|
||||
"zeppelin": {
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 510428,
|
||||
"deployment_gas": 94501114,
|
||||
"version": "af7ec04"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"bleeps": {
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 132165,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 39289935,
|
||||
"version": "bb90cd0"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 137869,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 38863224,
|
||||
"version": "bb90cd0"
|
||||
}
|
||||
},
|
||||
"colony": {
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 363606,
|
||||
"deployment_gas": null,
|
||||
"method_gas": null,
|
||||
"version": "573399b"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 363606,
|
||||
"deployment_gas": null,
|
||||
"method_gas": null,
|
||||
"version": "573399b"
|
||||
}
|
||||
},
|
||||
"elementfi": {
|
||||
"legacy-no-optimize": {
|
||||
"bytecode_size": 890560,
|
||||
"deployment_gas": null,
|
||||
"method_gas": null,
|
||||
"version": "87f8b5e"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 536668,
|
||||
"version": "87f8b5e"
|
||||
},
|
||||
"legacy-optimize-evm-only": {},
|
||||
"ir-no-optimize": {
|
||||
"bytecode_size": null,
|
||||
"method_gas": 2777867251,
|
||||
"version": "87f8b5e"
|
||||
}
|
||||
},
|
||||
"euler": {
|
||||
"ir-no-optimize": {
|
||||
"bytecode_size": 323909,
|
||||
"deployment_gas": 61591870,
|
||||
"method_gas": 3452105184,
|
||||
"version": "2ef99fc"
|
||||
},
|
||||
"legacy-no-optimize": {
|
||||
"bytecode_size": 323909,
|
||||
"deployment_gas": 61591870,
|
||||
"method_gas": 3452105184,
|
||||
"version": "c23e8bd"
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 182190,
|
||||
"deployment_gas": 35236828,
|
||||
"method_gas": 2777867251,
|
||||
"version": "c23e8bd"
|
||||
},
|
||||
"legacy-optimize-evm-only": {
|
||||
"bytecode_size": 202106,
|
||||
"deployment_gas": 38790600,
|
||||
"method_gas": 2907368790,
|
||||
"version": "v1.2.3"
|
||||
},
|
||||
"ir-optimize-evm-only": {
|
||||
"bytecode_size": 182190,
|
||||
"deployment_gas": 35236828,
|
||||
"method_gas": 2777867251
|
||||
},
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 182190,
|
||||
"deployment_gas": 35236828,
|
||||
"method_gas": 2777867251
|
||||
}
|
||||
},
|
||||
"ens": {
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 156937,
|
||||
"deployment_gas": 30073789,
|
||||
"method_gas": 105365362,
|
||||
"version": "v0.0.8"
|
||||
}
|
||||
},
|
||||
"zeppelin": {
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 510428,
|
||||
"deployment_gas": 94501114,
|
||||
"version": "af7ec04"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from textwrap import dedent
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from unittest_helpers import FIXTURE_DIR, load_fixture
|
||||
|
||||
# NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
|
||||
# pragma pylint: disable=import-error
|
||||
from externalTests.benchmark_diff import BenchmarkDiffer, DifferenceStyle, DiffTableSet, DiffTableFormatter, OutputFormat
|
||||
# pragma pylint: enable=import-error
|
||||
|
||||
SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-develop.json'
|
||||
SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-branch.json'
|
||||
|
||||
SUMMARIZED_DIFF_HUMANIZED_MD_PATH = FIXTURE_DIR / 'summarized-benchmark-diff-develop-branch-humanized.md'
|
||||
SUMMARIZED_DIFF_HUMANIZED_MD = load_fixture(SUMMARIZED_DIFF_HUMANIZED_MD_PATH)
|
||||
|
||||
|
||||
class TestBenchmarkDiff(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.maxDiff = 10000
|
||||
|
||||
def test_benchmark_diff(self):
|
||||
report_before = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH))
|
||||
report_after = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH))
|
||||
expected_diff = {
|
||||
"bleeps": {
|
||||
"ir-optimize-evm+yul": {
|
||||
# Numerical difference -> negative/positive/zero.
|
||||
# Zeros are not skipped to differentiate them from missing values.
|
||||
"bytecode_size": 132868 - 132165,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 39289198 - 39289935,
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
# No differences within preset -> zeros still present.
|
||||
"bytecode_size": 0,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 0,
|
||||
},
|
||||
},
|
||||
"colony": {
|
||||
# Preset missing on one side -> replace dict with string
|
||||
"ir-optimize-evm+yul": "!A",
|
||||
"legacy-no-optimize": "!B",
|
||||
"legacy-optimize-evm+yul": {
|
||||
"bytecode_size": 0,
|
||||
# Attribute missing on both sides -> skip
|
||||
#"deployment_gas":
|
||||
#"method_gas":
|
||||
},
|
||||
},
|
||||
"elementfi": {
|
||||
"legacy-no-optimize": {
|
||||
# Attributes null on one side -> replace value with string
|
||||
"bytecode_size": "!A",
|
||||
"deployment_gas": "!B",
|
||||
# Attribute null on both sides -> skip
|
||||
#"method_gas":
|
||||
},
|
||||
"legacy-optimize-evm+yul": {
|
||||
# Attributes missing on one side -> replace value with string
|
||||
"bytecode_size": "!A",
|
||||
"deployment_gas": "!B",
|
||||
# Attribute missing on both sides -> skip
|
||||
#"method_gas":
|
||||
},
|
||||
"ir-no-optimize": {
|
||||
# Attributes missing on one side, null on the other -> skip
|
||||
#"bytecode_size":
|
||||
#"deployment_gas":
|
||||
"method_gas": 0,
|
||||
},
|
||||
# Empty preset missing on one side -> replace dict with string
|
||||
"legacy-optimize-evm-only": "!A",
|
||||
"ir-optimize-evm-only": "!B",
|
||||
},
|
||||
"euler": {
|
||||
# Matching versions -> show attributes, skip version
|
||||
"ir-no-optimize": {
|
||||
"bytecode_size": 328540 - 323909,
|
||||
"deployment_gas": 0,
|
||||
"method_gas": 3537419168 - 3452105184,
|
||||
},
|
||||
# Different versions, different values -> replace whole preset with string
|
||||
"legacy-no-optimize": "!V",
|
||||
# Different versions, same values -> replace whole preset with string
|
||||
"legacy-optimize-evm+yul": "!V",
|
||||
# Different versions (not a commit hash), different values -> replace whole preset with string
|
||||
"legacy-optimize-evm-only": "!V",
|
||||
# Version missing on one side -> replace whole preset with string
|
||||
"ir-optimize-evm-only": "!V",
|
||||
# Version missing on both sides -> assume same version
|
||||
"ir-optimize-evm+yul": {
|
||||
"bytecode_size": 205211 - 182190,
|
||||
"deployment_gas": 39459629 - 35236828,
|
||||
"method_gas": 0,
|
||||
},
|
||||
},
|
||||
"zeppelin": {
|
||||
"legacy-optimize-evm+yul": {
|
||||
# Whole project identical -> attributes still present, with zeros
|
||||
"bytecode_size": 0,
|
||||
"deployment_gas": 0,
|
||||
# Field missing on both sides -> skip
|
||||
#"method_gas":
|
||||
}
|
||||
},
|
||||
# Empty project missing on one side -> replace its dict with a string
|
||||
"gnosis": "!B",
|
||||
"ens": "!A",
|
||||
}
|
||||
differ = BenchmarkDiffer(DifferenceStyle.ABSOLUTE, None, OutputFormat.JSON)
|
||||
self.assertEqual(differ.run(report_before, report_after), expected_diff)
|
||||
|
||||
|
||||
class TestBenchmarkDiffer(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.maxDiff = 10000
|
||||
|
||||
@staticmethod
|
||||
def _nest(value, levels):
|
||||
nested_value = value
|
||||
for level in levels:
|
||||
nested_value = {level: nested_value}
|
||||
|
||||
return nested_value
|
||||
|
||||
def _assert_single_value_diff_matches(self, differ, cases, nest_result=True, nestings=None):
|
||||
if nestings is None:
|
||||
nestings = [[], ['p'], ['p', 's'], ['p', 's', 'a']]
|
||||
|
||||
for levels in nestings:
|
||||
for (before, after, expected_diff) in cases:
|
||||
self.assertEqual(
|
||||
differ.run(self._nest(before, levels), self._nest(after, levels)),
|
||||
self._nest(expected_diff, levels) if nest_result else expected_diff,
|
||||
f'Wrong diff for {self._nest(before, levels)} vs {self._nest(after, levels)}'
|
||||
)
|
||||
|
||||
def test_empty(self):
|
||||
for style in DifferenceStyle:
|
||||
differ = BenchmarkDiffer(style, None, OutputFormat.JSON)
|
||||
self._assert_single_value_diff_matches(differ, [({}, {}, {})], nest_result=False)
|
||||
|
||||
def test_null(self):
|
||||
for style in DifferenceStyle:
|
||||
differ = BenchmarkDiffer(style, None, OutputFormat.JSON)
|
||||
self._assert_single_value_diff_matches(differ, [(None, None, {})], nest_result=False)
|
||||
|
||||
def test_number_diff_absolute_json(self):
|
||||
for output_format in OutputFormat:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(DifferenceStyle.ABSOLUTE, 4, output_format),
|
||||
[
|
||||
(2, 2, 0),
|
||||
(2, 5, 3),
|
||||
(5, 2, -3),
|
||||
(2.0, 2.0, 0),
|
||||
(2, 2.0, 0),
|
||||
(2.0, 2, 0),
|
||||
(2, 2.5, 2.5 - 2),
|
||||
(2.5, 2, 2 - 2.5),
|
||||
|
||||
(0, 0, 0),
|
||||
(0, 2, 2),
|
||||
(0, -2, -2),
|
||||
|
||||
(-3, -1, 2),
|
||||
(-1, -3, -2),
|
||||
(2, 0, -2),
|
||||
(-2, 0, 2),
|
||||
|
||||
(1.00006, 1, 1 - 1.00006),
|
||||
(1, 1.00006, 1.00006 - 1),
|
||||
(1.00004, 1, 1 - 1.00004),
|
||||
(1, 1.00004, 1.00004 - 1),
|
||||
],
|
||||
)
|
||||
|
||||
def test_number_diff_json(self):
|
||||
for output_format in OutputFormat:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(DifferenceStyle.RELATIVE, 4, output_format),
|
||||
[
|
||||
(2, 2, 0),
|
||||
(2, 5, (5 - 2) / 2),
|
||||
(5, 2, (2 - 5) / 5),
|
||||
(2.0, 2.0, 0),
|
||||
(2, 2.0, 0),
|
||||
(2.0, 2, 0),
|
||||
(2, 2.5, (2.5 - 2) / 2),
|
||||
(2.5, 2, (2 - 2.5) / 2.5),
|
||||
|
||||
(0, 0, 0),
|
||||
(0, 2, '+INF'),
|
||||
(0, -2, '-INF'),
|
||||
|
||||
(-3, -1, 0.6667),
|
||||
(-1, -3, -2),
|
||||
(2, 0, -1),
|
||||
(-2, 0, 1),
|
||||
|
||||
(1.00006, 1, -0.0001),
|
||||
(1, 1.00006, 0.0001),
|
||||
(1.000004, 1, '-0'),
|
||||
(1, 1.000004, '+0'),
|
||||
],
|
||||
)
|
||||
|
||||
def test_number_diff_humanized_json_and_console(self):
|
||||
for output_format in [OutputFormat.JSON, OutputFormat.CONSOLE]:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, output_format),
|
||||
[
|
||||
(2, 2, '0%'),
|
||||
(2, 5, '+150%'),
|
||||
(5, 2, '-60%'),
|
||||
(2.0, 2.0, '0%'),
|
||||
(2, 2.0, '0%'),
|
||||
(2.0, 2, '0%'),
|
||||
(2, 2.5, '+25%'),
|
||||
(2.5, 2, '-20%'),
|
||||
|
||||
(0, 0, '0%'),
|
||||
(0, 2, '+INF%'),
|
||||
(0, -2, '-INF%'),
|
||||
|
||||
(-3, -1, '+66.67%'),
|
||||
(-1, -3, '-200%'),
|
||||
(2, 0, '-100%'),
|
||||
(-2, 0, '+100%'),
|
||||
|
||||
(1.00006, 1, '-0.01%'),
|
||||
(1, 1.00006, '+0.01%'),
|
||||
(1.000004, 1, '-0%'),
|
||||
(1, 1.000004, '+0%'),
|
||||
],
|
||||
)
|
||||
|
||||
def test_number_diff_humanized_markdown(self):
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, OutputFormat.MARKDOWN),
|
||||
[
|
||||
(2, 2, '`0%`'),
|
||||
(2, 5, '**`+150% ❌`**'),
|
||||
(5, 2, '**`-60% ✅`**'),
|
||||
(2.0, 2.0, '`0%`'),
|
||||
(2, 2.0, '`0%`'),
|
||||
(2.0, 2, '`0%`'),
|
||||
(2, 2.5, '**`+25% ❌`**'),
|
||||
(2.5, 2, '**`-20% ✅`**'),
|
||||
|
||||
(0, 0, '`0%`'),
|
||||
(0, 2, '`+INF%`'),
|
||||
(0, -2, '`-INF%`'),
|
||||
|
||||
(-3, -1, '**`+66.67% ❌`**'),
|
||||
(-1, -3, '**`-200% ✅`**'),
|
||||
(2, 0, '**`-100% ✅`**'),
|
||||
(-2, 0, '**`+100% ❌`**'),
|
||||
|
||||
(1.00006, 1, '**`-0.01% ✅`**'),
|
||||
(1, 1.00006, '**`+0.01% ❌`**'),
|
||||
(1.000004, 1, '`-0%`'),
|
||||
(1, 1.000004, '`+0%`'),
|
||||
],
|
||||
)
|
||||
|
||||
def test_type_mismatch(self):
|
||||
for style in DifferenceStyle:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(style, 4, OutputFormat.JSON),
|
||||
[
|
||||
(1, {}, '!T'),
|
||||
({}, 1, '!T'),
|
||||
(1.5, {}, '!T'),
|
||||
({}, 1.5, '!T'),
|
||||
('1', {}, '!T'),
|
||||
({}, '1', '!T'),
|
||||
(1, '1', '!T'),
|
||||
('1', 1, '!T'),
|
||||
(1.5, '1', '!T'),
|
||||
('1', 1.5, '!T'),
|
||||
('1', '1', '!T'),
|
||||
],
|
||||
)
|
||||
|
||||
def test_version_mismatch(self):
|
||||
for style in DifferenceStyle:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(style, 4, OutputFormat.JSON),
|
||||
[
|
||||
({'a': 123, 'version': 1}, {'a': 123, 'version': 2}, '!V'),
|
||||
({'a': 123, 'version': 2}, {'a': 123, 'version': 1}, '!V'),
|
||||
({'a': 123, 'version': 'a'}, {'a': 123, 'version': 'b'}, '!V'),
|
||||
({'a': 123, 'version': 'a'}, {'a': 123, 'version': 1}, '!V'),
|
||||
|
||||
({'a': 'a', 'version': 1}, {'a': 'a', 'version': 2}, '!V'),
|
||||
({'a': {}, 'version': 1}, {'a': {}, 'version': 2}, '!V'),
|
||||
({'s': {'a': 1}, 'version': 1}, {'s': {'a': 1}, 'version': 2}, '!V'),
|
||||
|
||||
({'a': 123, 'version': 1}, {'a': 456, 'version': 2}, '!V'),
|
||||
({'a': 'a', 'version': 1}, {'a': 'b', 'version': 2}, '!V'),
|
||||
({'s': {'a': 1}, 'version': 1}, {'s': {'a': 2}, 'version': 2}, '!V'),
|
||||
],
|
||||
)
|
||||
|
||||
def test_missing(self):
|
||||
for style in DifferenceStyle:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(style, None, OutputFormat.JSON),
|
||||
[
|
||||
(1, None, '!A'),
|
||||
(None, 1, '!B'),
|
||||
('1', None, '!A'),
|
||||
(None, '1', '!B'),
|
||||
({}, None, '!A'),
|
||||
(None, {}, '!B'),
|
||||
|
||||
({'x': 1}, {}, {'x': '!A'}),
|
||||
({}, {'x': 1}, {'x': '!B'}),
|
||||
({'x': 1}, {'x': None}, {'x': '!A'}),
|
||||
({'x': None}, {'x': 1}, {'x': '!B'}),
|
||||
({'x': 1}, {'y': 1}, {'x': '!A', 'y': '!B'}),
|
||||
|
||||
({'x': {}}, {}, {'x': '!A'}),
|
||||
({}, {'x': {}}, {'x': '!B'}),
|
||||
({'p': {'x': {}}}, {}, {'p': '!A'}),
|
||||
({}, {'p': {'x': {}}}, {'p': '!B'}),
|
||||
],
|
||||
)
|
||||
|
||||
def test_missing_vs_null(self):
|
||||
for style in DifferenceStyle:
|
||||
self._assert_single_value_diff_matches(
|
||||
BenchmarkDiffer(style, None, OutputFormat.JSON),
|
||||
[
|
||||
({'a': None}, {}, {}),
|
||||
({}, {'a': None}, {}),
|
||||
],
|
||||
nest_result=False,
|
||||
)
|
||||
|
||||
|
||||
class TestDiffTableFormatter(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.maxDiff = 10000
|
||||
|
||||
self.report_before = {
|
||||
'project A': {
|
||||
'preset X': {'A1': 99, 'A2': 50, 'version': 1},
|
||||
'preset Y': {'A1': 0, 'A2': 50, 'version': 1},
|
||||
},
|
||||
'project B': {
|
||||
'preset X': { 'A2': 50},
|
||||
'preset Y': {'A1': 0},
|
||||
},
|
||||
'project C': {
|
||||
'preset X': {'A1': 0, 'A2': 50, 'version': 1},
|
||||
},
|
||||
'project D': {
|
||||
'preset X': {'A1': 999},
|
||||
},
|
||||
}
|
||||
self.report_after = {
|
||||
'project A': {
|
||||
'preset X': {'A1': 100, 'A2': 50, 'version': 1},
|
||||
'preset Y': {'A1': 500, 'A2': 500, 'version': 2},
|
||||
},
|
||||
'project B': {
|
||||
'preset X': {'A1': 0},
|
||||
'preset Y': { 'A2': 50},
|
||||
},
|
||||
'project C': {
|
||||
'preset Y': {'A1': 0, 'A2': 50, 'version': 1},
|
||||
},
|
||||
'project E': {
|
||||
'preset Y': { 'A2': 999},
|
||||
},
|
||||
}
|
||||
|
||||
def test_diff_table_formatter(self):
|
||||
report_before = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH))
|
||||
report_after = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH))
|
||||
differ = BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, OutputFormat.MARKDOWN)
|
||||
diff = differ.run(report_before, report_after)
|
||||
|
||||
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.MARKDOWN), SUMMARIZED_DIFF_HUMANIZED_MD)
|
||||
|
||||
def test_diff_table_formatter_json_absolute(self):
|
||||
differ = BenchmarkDiffer(DifferenceStyle.ABSOLUTE, 4, OutputFormat.JSON)
|
||||
diff = differ.run(self.report_before, self.report_after)
|
||||
|
||||
expected_formatted_table = dedent("""\
|
||||
{
|
||||
"preset X": {
|
||||
"project A": {
|
||||
"A1": 1,
|
||||
"A2": 0
|
||||
},
|
||||
"project B": {
|
||||
"A1": "!B",
|
||||
"A2": "!A"
|
||||
},
|
||||
"project C": {
|
||||
"A1": "!A",
|
||||
"A2": "!A"
|
||||
},
|
||||
"project D": {
|
||||
"A1": "!A",
|
||||
"A2": "!A"
|
||||
},
|
||||
"project E": {
|
||||
"A1": "!B",
|
||||
"A2": "!B"
|
||||
}
|
||||
},
|
||||
"preset Y": {
|
||||
"project A": {
|
||||
"A1": "!V",
|
||||
"A2": "!V"
|
||||
},
|
||||
"project B": {
|
||||
"A1": "!A",
|
||||
"A2": "!B"
|
||||
},
|
||||
"project C": {
|
||||
"A1": "!B",
|
||||
"A2": "!B"
|
||||
},
|
||||
"project D": {
|
||||
"A1": "!A",
|
||||
"A2": "!A"
|
||||
},
|
||||
"project E": {
|
||||
"A1": "!B",
|
||||
"A2": "!B"
|
||||
}
|
||||
}
|
||||
}"""
|
||||
)
|
||||
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.JSON), expected_formatted_table)
|
||||
|
||||
def test_diff_table_formatter_console_relative(self):
|
||||
differ = BenchmarkDiffer(DifferenceStyle.RELATIVE, 4, OutputFormat.CONSOLE)
|
||||
diff = differ.run(self.report_before, self.report_after)
|
||||
|
||||
expected_formatted_table = dedent("""
|
||||
PRESET X
|
||||
|-----------|--------|----|
|
||||
| project | A1 | A2 |
|
||||
|-----------|--------|----|
|
||||
| project A | 0.0101 | 0 |
|
||||
| project B | !B | !A |
|
||||
| project C | !A | !A |
|
||||
| project D | !A | !A |
|
||||
| project E | !B | !B |
|
||||
|-----------|--------|----|
|
||||
|
||||
PRESET Y
|
||||
|-----------|----|----|
|
||||
| project | A1 | A2 |
|
||||
|-----------|----|----|
|
||||
| project A | !V | !V |
|
||||
| project B | !A | !B |
|
||||
| project C | !B | !B |
|
||||
| project D | !A | !A |
|
||||
| project E | !B | !B |
|
||||
|-----------|----|----|
|
||||
""")
|
||||
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.CONSOLE), expected_formatted_table)
|
||||
|
||||
def test_diff_table_formatter_markdown_humanized(self):
|
||||
differ = BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4, OutputFormat.MARKDOWN)
|
||||
diff = differ.run(self.report_before, self.report_after)
|
||||
|
||||
expected_formatted_table = dedent("""
|
||||
### `preset X`
|
||||
| project | A1 | A2 |
|
||||
|:---------:|---------------:|-----:|
|
||||
| project A | **`+1.01% ❌`** | `0%` |
|
||||
| project B | `!B` | `!A` |
|
||||
| project C | `!A` | `!A` |
|
||||
| project D | `!A` | `!A` |
|
||||
| project E | `!B` | `!B` |
|
||||
|
||||
### `preset Y`
|
||||
| project | A1 | A2 |
|
||||
|:---------:|-----:|-----:|
|
||||
| project A | `!V` | `!V` |
|
||||
| project B | `!A` | `!B` |
|
||||
| project C | `!B` | `!B` |
|
||||
| project D | `!A` | `!A` |
|
||||
| project E | `!B` | `!B` |
|
||||
|
||||
|
||||
`!V` = version mismatch
|
||||
`!B` = no value in the "before" version
|
||||
`!A` = no value in the "after" version
|
||||
`!T` = one or both values were not numeric and could not be compared
|
||||
`-0` = very small negative value rounded to zero
|
||||
`+0` = very small positive value rounded to zero
|
||||
|
||||
""")
|
||||
self.assertEqual(DiffTableFormatter.run(DiffTableSet(diff), OutputFormat.MARKDOWN), expected_formatted_table)
|
||||
Reference in New Issue
Block a user