mirror of
https://github.com/ethereum/solidity
synced 2023-10-03 13:03:40 +00:00
benchmark_diff: A Python script for diffing summarized benchmarks from external tests
This commit is contained in:
Kamil Śliwak
parent
0944e6853f
commit
ee5e878ad7
212
scripts/externalTests/benchmark_diff.py
Executable file
212
scripts/externalTests/benchmark_diff.py
Executable file
@ -0,0 +1,212 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Optional, Union
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
class DifferenceStyle(Enum):
|
||||||
|
ABSOLUTE = 'absolute'
|
||||||
|
RELATIVE = 'relative'
|
||||||
|
HUMANIZED = 'humanized'
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_RELATIVE_PRECISION = 4
|
||||||
|
DEFAULT_DIFFERENCE_STYLE = DifferenceStyle.ABSOLUTE
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class CommandLineError(ValidationError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class BenchmarkDiffer:
|
||||||
|
difference_style: DifferenceStyle
|
||||||
|
relative_precision: Optional[int]
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
difference_style: DifferenceStyle,
|
||||||
|
relative_precision: Optional[int],
|
||||||
|
):
|
||||||
|
self.difference_style = difference_style
|
||||||
|
self.relative_precision = relative_precision
|
||||||
|
|
||||||
|
def run(self, before: Any, after: Any) -> Optional[Union[dict, str, int, float]]:
|
||||||
|
if not isinstance(before, dict) or not isinstance(after, dict):
|
||||||
|
return self._diff_scalars(before, after)
|
||||||
|
|
||||||
|
if before.get('version') != after.get('version'):
|
||||||
|
return self._humanize_diff('!V')
|
||||||
|
|
||||||
|
diff = {}
|
||||||
|
for key in (set(before) | set(after)) - {'version'}:
|
||||||
|
value_diff = self.run(before.get(key), after.get(key))
|
||||||
|
if value_diff not in [None, {}]:
|
||||||
|
diff[key] = value_diff
|
||||||
|
|
||||||
|
return diff
|
||||||
|
|
||||||
|
def _diff_scalars(self, before: Any, after: Any) -> Optional[Union[str, int, float, dict]]:
|
||||||
|
assert not isinstance(before, dict) or not isinstance(after, dict)
|
||||||
|
|
||||||
|
if before is None and after is None:
|
||||||
|
return {}
|
||||||
|
if before is None:
|
||||||
|
return self._humanize_diff('!B')
|
||||||
|
if after is None:
|
||||||
|
return self._humanize_diff('!A')
|
||||||
|
if not isinstance(before, (int, float)) or not isinstance(after, (int, float)):
|
||||||
|
return self._humanize_diff('!T')
|
||||||
|
|
||||||
|
number_diff = self._diff_numbers(before, after)
|
||||||
|
if self.difference_style != DifferenceStyle.HUMANIZED:
|
||||||
|
return number_diff
|
||||||
|
|
||||||
|
return self._humanize_diff(number_diff)
|
||||||
|
|
||||||
|
def _diff_numbers(self, value_before: Union[int, float], value_after: Union[int, float]) -> Union[str, int, float]:
|
||||||
|
diff: Union[str, int, float]
|
||||||
|
|
||||||
|
if self.difference_style == DifferenceStyle.ABSOLUTE:
|
||||||
|
diff = value_after - value_before
|
||||||
|
if isinstance(diff, float) and diff.is_integer():
|
||||||
|
diff = int(diff)
|
||||||
|
|
||||||
|
return diff
|
||||||
|
|
||||||
|
if value_before == 0:
|
||||||
|
if value_after > 0:
|
||||||
|
return '+INF'
|
||||||
|
elif value_after < 0:
|
||||||
|
return '-INF'
|
||||||
|
else:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
diff = (value_after - value_before) / abs(value_before)
|
||||||
|
if self.relative_precision is not None:
|
||||||
|
rounded_diff = round(diff, self.relative_precision)
|
||||||
|
if rounded_diff == 0 and diff < 0:
|
||||||
|
diff = '-0'
|
||||||
|
elif rounded_diff == 0 and diff > 0:
|
||||||
|
diff = '+0'
|
||||||
|
else:
|
||||||
|
diff = rounded_diff
|
||||||
|
|
||||||
|
if isinstance(diff, float) and diff.is_integer():
|
||||||
|
diff = int(diff)
|
||||||
|
|
||||||
|
return diff
|
||||||
|
|
||||||
|
def _humanize_diff(self, diff: Union[str, int, float]) -> str:
|
||||||
|
if isinstance(diff, str) and diff.startswith('!'):
|
||||||
|
return diff
|
||||||
|
|
||||||
|
value: Union[str, int, float]
|
||||||
|
if isinstance(diff, (int, float)):
|
||||||
|
value = diff * 100
|
||||||
|
if isinstance(value, float) and self.relative_precision is not None:
|
||||||
|
# The multiplication can result in new significant digits appearing. We need to reround.
|
||||||
|
# NOTE: round() works fine with negative precision.
|
||||||
|
value = round(value, self.relative_precision - 2)
|
||||||
|
if isinstance(value, float) and value.is_integer():
|
||||||
|
value = int(value)
|
||||||
|
prefix = ''
|
||||||
|
if diff < 0:
|
||||||
|
prefix = ''
|
||||||
|
elif diff > 0:
|
||||||
|
prefix = '+'
|
||||||
|
else:
|
||||||
|
value = diff
|
||||||
|
prefix = ''
|
||||||
|
|
||||||
|
return f"{prefix}{value}%"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CommandLineOptions:
|
||||||
|
report_before: Path
|
||||||
|
report_after: Path
|
||||||
|
difference_style: DifferenceStyle
|
||||||
|
relative_precision: int
|
||||||
|
|
||||||
|
|
||||||
|
def process_commandline() -> CommandLineOptions:
|
||||||
|
script_description = (
|
||||||
|
"Compares summarized benchmark reports and outputs JSON with the same structure but listing only differences."
|
||||||
|
)
|
||||||
|
|
||||||
|
parser = ArgumentParser(description=script_description)
|
||||||
|
parser.add_argument(dest='report_before', help="Path to a JSON file containing original benchmark results.")
|
||||||
|
parser.add_argument(dest='report_after', help="Path to a JSON file containing new benchmark results.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--style',
|
||||||
|
dest='difference_style',
|
||||||
|
choices=[s.value for s in DifferenceStyle],
|
||||||
|
help=(
|
||||||
|
"How to present numeric differences: "
|
||||||
|
f"'{DifferenceStyle.ABSOLUTE.value}' subtracts new from original; "
|
||||||
|
f"'{DifferenceStyle.RELATIVE.value}' also divides by the original; "
|
||||||
|
f"'{DifferenceStyle.HUMANIZED.value}' is like relative but value is a percentage and "
|
||||||
|
"positive/negative changes are emphasized. "
|
||||||
|
f"(default: '{DEFAULT_DIFFERENCE_STYLE}')."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# NOTE: Negative values are valid for precision. round() handles them in a sensible way.
|
||||||
|
parser.add_argument(
|
||||||
|
'--precision',
|
||||||
|
dest='relative_precision',
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_RELATIVE_PRECISION,
|
||||||
|
help=(
|
||||||
|
"Number of significant digits for relative differences. "
|
||||||
|
f"Note that with --style={DifferenceStyle.HUMANIZED.value} the rounding is applied "
|
||||||
|
"**before** converting the value to a percentage so you need to add 2. "
|
||||||
|
f"Has no effect when used together with --style={DifferenceStyle.ABSOLUTE.value}. "
|
||||||
|
f"(default: {DEFAULT_RELATIVE_PRECISION})"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
options = parser.parse_args()
|
||||||
|
|
||||||
|
if options.difference_style is not None:
|
||||||
|
difference_style = DifferenceStyle(options.difference_style)
|
||||||
|
else:
|
||||||
|
difference_style = DEFAULT_DIFFERENCE_STYLE
|
||||||
|
|
||||||
|
processed_options = CommandLineOptions(
|
||||||
|
report_before=Path(options.report_before),
|
||||||
|
report_after=Path(options.report_after),
|
||||||
|
difference_style=difference_style,
|
||||||
|
relative_precision=options.relative_precision,
|
||||||
|
)
|
||||||
|
|
||||||
|
return processed_options
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
try:
|
||||||
|
options = process_commandline()
|
||||||
|
|
||||||
|
differ = BenchmarkDiffer(options.difference_style, options.relative_precision)
|
||||||
|
diff = differ.run(
|
||||||
|
json.loads(options.report_before.read_text('utf-8')),
|
||||||
|
json.loads(options.report_after.read_text('utf-8')),
|
||||||
|
)
|
||||||
|
|
||||||
|
print(json.dumps(diff, indent=4, sort_keys=True))
|
||||||
|
|
||||||
|
return 0
|
||||||
|
except CommandLineError as exception:
|
||||||
|
print(f"ERROR: {exception}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
100
test/scripts/fixtures/summarized-benchmarks-branch.json
Normal file
100
test/scripts/fixtures/summarized-benchmarks-branch.json
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
{
|
||||||
|
"bleeps": {
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 132868,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 39289198,
|
||||||
|
"version": "bb90cd0"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 137869,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 38863224,
|
||||||
|
"version": "bb90cd0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colony": {
|
||||||
|
"legacy-no-optimize": {
|
||||||
|
"bytecode_size": 664190,
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "573399b"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 363606,
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "573399b"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"elementfi": {
|
||||||
|
"legacy-no-optimize": {
|
||||||
|
"bytecode_size": null,
|
||||||
|
"deployment_gas": 69200158,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "87f8b5e"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"deployment_gas": 40951128,
|
||||||
|
"version": "87f8b5e"
|
||||||
|
},
|
||||||
|
"ir-optimize-evm-only": {},
|
||||||
|
"ir-no-optimize": {
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": 2777867251,
|
||||||
|
"version": "87f8b5e"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"euler": {
|
||||||
|
"ir-no-optimize": {
|
||||||
|
"bytecode_size": 328540,
|
||||||
|
"deployment_gas": 61591870,
|
||||||
|
"method_gas": 3537419168,
|
||||||
|
"version": "2ef99fc"
|
||||||
|
},
|
||||||
|
"legacy-no-optimize": {
|
||||||
|
"bytecode_size": 328540,
|
||||||
|
"deployment_gas": 62590688,
|
||||||
|
"method_gas": 3537419168,
|
||||||
|
"version": "2ef99fc"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 182190,
|
||||||
|
"deployment_gas": 35236828,
|
||||||
|
"method_gas": 2777867251,
|
||||||
|
"version": "2ef99fc"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm-only": {
|
||||||
|
"bytecode_size": 205211,
|
||||||
|
"deployment_gas": 39459629,
|
||||||
|
"method_gas": 2978467272,
|
||||||
|
"version": "2ef99fc"
|
||||||
|
},
|
||||||
|
"ir-optimize-evm-only": {
|
||||||
|
"bytecode_size": 205211,
|
||||||
|
"deployment_gas": 39459629,
|
||||||
|
"method_gas": 2978467272,
|
||||||
|
"version": "2ef99fc"
|
||||||
|
},
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 205211,
|
||||||
|
"deployment_gas": 39459629,
|
||||||
|
"method_gas": 2777867251
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gnosis": {
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 56069,
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "ea09294"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"zeppelin": {
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 510428,
|
||||||
|
"deployment_gas": 94501114,
|
||||||
|
"version": "af7ec04"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
99
test/scripts/fixtures/summarized-benchmarks-develop.json
Normal file
99
test/scripts/fixtures/summarized-benchmarks-develop.json
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
{
|
||||||
|
"bleeps": {
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 132165,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 39289935,
|
||||||
|
"version": "bb90cd0"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 137869,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 38863224,
|
||||||
|
"version": "bb90cd0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"colony": {
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 363606,
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "573399b"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 363606,
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "573399b"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"elementfi": {
|
||||||
|
"legacy-no-optimize": {
|
||||||
|
"bytecode_size": 890560,
|
||||||
|
"deployment_gas": null,
|
||||||
|
"method_gas": null,
|
||||||
|
"version": "87f8b5e"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 536668,
|
||||||
|
"version": "87f8b5e"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm-only": {},
|
||||||
|
"ir-no-optimize": {
|
||||||
|
"bytecode_size": null,
|
||||||
|
"method_gas": 2777867251,
|
||||||
|
"version": "87f8b5e"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"euler": {
|
||||||
|
"ir-no-optimize": {
|
||||||
|
"bytecode_size": 323909,
|
||||||
|
"deployment_gas": 61591870,
|
||||||
|
"method_gas": 3452105184,
|
||||||
|
"version": "2ef99fc"
|
||||||
|
},
|
||||||
|
"legacy-no-optimize": {
|
||||||
|
"bytecode_size": 323909,
|
||||||
|
"deployment_gas": 61591870,
|
||||||
|
"method_gas": 3452105184,
|
||||||
|
"version": "c23e8bd"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 182190,
|
||||||
|
"deployment_gas": 35236828,
|
||||||
|
"method_gas": 2777867251,
|
||||||
|
"version": "c23e8bd"
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm-only": {
|
||||||
|
"bytecode_size": 202106,
|
||||||
|
"deployment_gas": 38790600,
|
||||||
|
"method_gas": 2907368790,
|
||||||
|
"version": "v1.2.3"
|
||||||
|
},
|
||||||
|
"ir-optimize-evm-only": {
|
||||||
|
"bytecode_size": 182190,
|
||||||
|
"deployment_gas": 35236828,
|
||||||
|
"method_gas": 2777867251
|
||||||
|
},
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 182190,
|
||||||
|
"deployment_gas": 35236828,
|
||||||
|
"method_gas": 2777867251
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ens": {
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 156937,
|
||||||
|
"deployment_gas": 30073789,
|
||||||
|
"method_gas": 105365362,
|
||||||
|
"version": "v0.0.8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"zeppelin": {
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 510428,
|
||||||
|
"deployment_gas": 94501114,
|
||||||
|
"version": "af7ec04"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
309
test/scripts/test_externalTests_benchmark_diff.py
Normal file
309
test/scripts/test_externalTests_benchmark_diff.py
Normal file
@ -0,0 +1,309 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from unittest_helpers import FIXTURE_DIR, load_fixture
|
||||||
|
|
||||||
|
# NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports
|
||||||
|
# pragma pylint: disable=import-error
|
||||||
|
from externalTests.benchmark_diff import BenchmarkDiffer, DifferenceStyle
|
||||||
|
# pragma pylint: enable=import-error
|
||||||
|
|
||||||
|
SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-develop.json'
|
||||||
|
SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH = FIXTURE_DIR / 'summarized-benchmarks-branch.json'
|
||||||
|
|
||||||
|
|
||||||
|
class TestBenchmarkDiff(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.maxDiff = 10000
|
||||||
|
|
||||||
|
def test_benchmark_diff(self):
|
||||||
|
report_before = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_DEVELOP_JSON_PATH))
|
||||||
|
report_after = json.loads(load_fixture(SUMMARIZED_BENCHMARKS_BRANCH_JSON_PATH))
|
||||||
|
expected_diff = {
|
||||||
|
"bleeps": {
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
# Numerical difference -> negative/positive/zero.
|
||||||
|
# Zeros are not skipped to differentiate them from missing values.
|
||||||
|
"bytecode_size": 132868 - 132165,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 39289198 - 39289935,
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
# No differences within preset -> zeros still present.
|
||||||
|
"bytecode_size": 0,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"colony": {
|
||||||
|
# Preset missing on one side -> replace dict with string
|
||||||
|
"ir-optimize-evm+yul": "!A",
|
||||||
|
"legacy-no-optimize": "!B",
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 0,
|
||||||
|
# Attribute missing on both sides -> skip
|
||||||
|
#"deployment_gas":
|
||||||
|
#"method_gas":
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"elementfi": {
|
||||||
|
"legacy-no-optimize": {
|
||||||
|
# Attributes null on one side -> replace value with string
|
||||||
|
"bytecode_size": "!A",
|
||||||
|
"deployment_gas": "!B",
|
||||||
|
# Attribute null on both sides -> skip
|
||||||
|
#"method_gas":
|
||||||
|
},
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
# Attributes missing on one side -> replace value with string
|
||||||
|
"bytecode_size": "!A",
|
||||||
|
"deployment_gas": "!B",
|
||||||
|
# Attribute missing on both sides -> skip
|
||||||
|
#"method_gas":
|
||||||
|
},
|
||||||
|
"ir-no-optimize": {
|
||||||
|
# Attributes missing on one side, null on the other -> skip
|
||||||
|
#"bytecode_size":
|
||||||
|
#"deployment_gas":
|
||||||
|
"method_gas": 0,
|
||||||
|
},
|
||||||
|
# Empty preset missing on one side -> replace dict with string
|
||||||
|
"legacy-optimize-evm-only": "!A",
|
||||||
|
"ir-optimize-evm-only": "!B",
|
||||||
|
},
|
||||||
|
"euler": {
|
||||||
|
# Matching versions -> show attributes, skip version
|
||||||
|
"ir-no-optimize": {
|
||||||
|
"bytecode_size": 328540 - 323909,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
"method_gas": 3537419168 - 3452105184,
|
||||||
|
},
|
||||||
|
# Different versions, different values -> replace whole preset with string
|
||||||
|
"legacy-no-optimize": "!V",
|
||||||
|
# Different versions, same values -> replace whole preset with string
|
||||||
|
"legacy-optimize-evm+yul": "!V",
|
||||||
|
# Different versions (not a commit hash), different values -> replace whole preset with string
|
||||||
|
"legacy-optimize-evm-only": "!V",
|
||||||
|
# Version missing on one side -> replace whole preset with string
|
||||||
|
"ir-optimize-evm-only": "!V",
|
||||||
|
# Version missing on both sides -> assume same version
|
||||||
|
"ir-optimize-evm+yul": {
|
||||||
|
"bytecode_size": 205211 - 182190,
|
||||||
|
"deployment_gas": 39459629 - 35236828,
|
||||||
|
"method_gas": 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"zeppelin": {
|
||||||
|
"legacy-optimize-evm+yul": {
|
||||||
|
# Whole project identical -> attributes still present, with zeros
|
||||||
|
"bytecode_size": 0,
|
||||||
|
"deployment_gas": 0,
|
||||||
|
# Field missing on both sides -> skip
|
||||||
|
#"method_gas":
|
||||||
|
}
|
||||||
|
},
|
||||||
|
# Empty project missing on one side -> replace its dict with a string
|
||||||
|
"gnosis": "!B",
|
||||||
|
"ens": "!A",
|
||||||
|
}
|
||||||
|
differ = BenchmarkDiffer(DifferenceStyle.ABSOLUTE, None)
|
||||||
|
self.assertEqual(differ.run(report_before, report_after), expected_diff)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBenchmarkDiffer(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.maxDiff = 10000
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _nest(value, levels):
|
||||||
|
nested_value = value
|
||||||
|
for level in levels:
|
||||||
|
nested_value = {level: nested_value}
|
||||||
|
|
||||||
|
return nested_value
|
||||||
|
|
||||||
|
def _assert_single_value_diff_matches(self, differ, cases, nest_result=True, nestings=None):
|
||||||
|
if nestings is None:
|
||||||
|
nestings = [[], ['p'], ['p', 's'], ['p', 's', 'a']]
|
||||||
|
|
||||||
|
for levels in nestings:
|
||||||
|
for (before, after, expected_diff) in cases:
|
||||||
|
self.assertEqual(
|
||||||
|
differ.run(self._nest(before, levels), self._nest(after, levels)),
|
||||||
|
self._nest(expected_diff, levels) if nest_result else expected_diff,
|
||||||
|
f'Wrong diff for {self._nest(before, levels)} vs {self._nest(after, levels)}'
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
for style in DifferenceStyle:
|
||||||
|
differ = BenchmarkDiffer(style, None)
|
||||||
|
self._assert_single_value_diff_matches(differ, [({}, {}, {})], nest_result=False)
|
||||||
|
|
||||||
|
def test_null(self):
|
||||||
|
for style in DifferenceStyle:
|
||||||
|
differ = BenchmarkDiffer(style, None)
|
||||||
|
self._assert_single_value_diff_matches(differ, [(None, None, {})], nest_result=False)
|
||||||
|
|
||||||
|
def test_number_diff_absolute_json(self):
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(DifferenceStyle.ABSOLUTE, 4),
|
||||||
|
[
|
||||||
|
(2, 2, 0),
|
||||||
|
(2, 5, 3),
|
||||||
|
(5, 2, -3),
|
||||||
|
(2.0, 2.0, 0),
|
||||||
|
(2, 2.0, 0),
|
||||||
|
(2.0, 2, 0),
|
||||||
|
(2, 2.5, 2.5 - 2),
|
||||||
|
(2.5, 2, 2 - 2.5),
|
||||||
|
|
||||||
|
(0, 0, 0),
|
||||||
|
(0, 2, 2),
|
||||||
|
(0, -2, -2),
|
||||||
|
|
||||||
|
(-3, -1, 2),
|
||||||
|
(-1, -3, -2),
|
||||||
|
(2, 0, -2),
|
||||||
|
(-2, 0, 2),
|
||||||
|
|
||||||
|
(1.00006, 1, 1 - 1.00006),
|
||||||
|
(1, 1.00006, 1.00006 - 1),
|
||||||
|
(1.00004, 1, 1 - 1.00004),
|
||||||
|
(1, 1.00004, 1.00004 - 1),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_number_diff_json(self):
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(DifferenceStyle.RELATIVE, 4),
|
||||||
|
[
|
||||||
|
(2, 2, 0),
|
||||||
|
(2, 5, (5 - 2) / 2),
|
||||||
|
(5, 2, (2 - 5) / 5),
|
||||||
|
(2.0, 2.0, 0),
|
||||||
|
(2, 2.0, 0),
|
||||||
|
(2.0, 2, 0),
|
||||||
|
(2, 2.5, (2.5 - 2) / 2),
|
||||||
|
(2.5, 2, (2 - 2.5) / 2.5),
|
||||||
|
|
||||||
|
(0, 0, 0),
|
||||||
|
(0, 2, '+INF'),
|
||||||
|
(0, -2, '-INF'),
|
||||||
|
|
||||||
|
(-3, -1, 0.6667),
|
||||||
|
(-1, -3, -2),
|
||||||
|
(2, 0, -1),
|
||||||
|
(-2, 0, 1),
|
||||||
|
|
||||||
|
(1.00006, 1, -0.0001),
|
||||||
|
(1, 1.00006, 0.0001),
|
||||||
|
(1.000004, 1, '-0'),
|
||||||
|
(1, 1.000004, '+0'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_number_diff_humanized_json(self):
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(DifferenceStyle.HUMANIZED, 4),
|
||||||
|
[
|
||||||
|
(2, 2, '0%'),
|
||||||
|
(2, 5, '+150%'),
|
||||||
|
(5, 2, '-60%'),
|
||||||
|
(2.0, 2.0, '0%'),
|
||||||
|
(2, 2.0, '0%'),
|
||||||
|
(2.0, 2, '0%'),
|
||||||
|
(2, 2.5, '+25%'),
|
||||||
|
(2.5, 2, '-20%'),
|
||||||
|
|
||||||
|
(0, 0, '0%'),
|
||||||
|
(0, 2, '+INF%'),
|
||||||
|
(0, -2, '-INF%'),
|
||||||
|
|
||||||
|
(-3, -1, '+66.67%'),
|
||||||
|
(-1, -3, '-200%'),
|
||||||
|
(2, 0, '-100%'),
|
||||||
|
(-2, 0, '+100%'),
|
||||||
|
|
||||||
|
(1.00006, 1, '-0.01%'),
|
||||||
|
(1, 1.00006, '+0.01%'),
|
||||||
|
(1.000004, 1, '-0%'),
|
||||||
|
(1, 1.000004, '+0%'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_type_mismatch(self):
|
||||||
|
for style in DifferenceStyle:
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(style, 4),
|
||||||
|
[
|
||||||
|
(1, {}, '!T'),
|
||||||
|
({}, 1, '!T'),
|
||||||
|
(1.5, {}, '!T'),
|
||||||
|
({}, 1.5, '!T'),
|
||||||
|
('1', {}, '!T'),
|
||||||
|
({}, '1', '!T'),
|
||||||
|
(1, '1', '!T'),
|
||||||
|
('1', 1, '!T'),
|
||||||
|
(1.5, '1', '!T'),
|
||||||
|
('1', 1.5, '!T'),
|
||||||
|
('1', '1', '!T'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_version_mismatch(self):
|
||||||
|
for style in DifferenceStyle:
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(style, 4),
|
||||||
|
[
|
||||||
|
({'a': 123, 'version': 1}, {'a': 123, 'version': 2}, '!V'),
|
||||||
|
({'a': 123, 'version': 2}, {'a': 123, 'version': 1}, '!V'),
|
||||||
|
({'a': 123, 'version': 'a'}, {'a': 123, 'version': 'b'}, '!V'),
|
||||||
|
({'a': 123, 'version': 'a'}, {'a': 123, 'version': 1}, '!V'),
|
||||||
|
|
||||||
|
({'a': 'a', 'version': 1}, {'a': 'a', 'version': 2}, '!V'),
|
||||||
|
({'a': {}, 'version': 1}, {'a': {}, 'version': 2}, '!V'),
|
||||||
|
({'s': {'a': 1}, 'version': 1}, {'s': {'a': 1}, 'version': 2}, '!V'),
|
||||||
|
|
||||||
|
({'a': 123, 'version': 1}, {'a': 456, 'version': 2}, '!V'),
|
||||||
|
({'a': 'a', 'version': 1}, {'a': 'b', 'version': 2}, '!V'),
|
||||||
|
({'s': {'a': 1}, 'version': 1}, {'s': {'a': 2}, 'version': 2}, '!V'),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_missing(self):
|
||||||
|
for style in DifferenceStyle:
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(style, None),
|
||||||
|
[
|
||||||
|
(1, None, '!A'),
|
||||||
|
(None, 1, '!B'),
|
||||||
|
('1', None, '!A'),
|
||||||
|
(None, '1', '!B'),
|
||||||
|
({}, None, '!A'),
|
||||||
|
(None, {}, '!B'),
|
||||||
|
|
||||||
|
({'x': 1}, {}, {'x': '!A'}),
|
||||||
|
({}, {'x': 1}, {'x': '!B'}),
|
||||||
|
({'x': 1}, {'x': None}, {'x': '!A'}),
|
||||||
|
({'x': None}, {'x': 1}, {'x': '!B'}),
|
||||||
|
({'x': 1}, {'y': 1}, {'x': '!A', 'y': '!B'}),
|
||||||
|
|
||||||
|
({'x': {}}, {}, {'x': '!A'}),
|
||||||
|
({}, {'x': {}}, {'x': '!B'}),
|
||||||
|
({'p': {'x': {}}}, {}, {'p': '!A'}),
|
||||||
|
({}, {'p': {'x': {}}}, {'p': '!B'}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_missing_vs_null(self):
|
||||||
|
for style in DifferenceStyle:
|
||||||
|
self._assert_single_value_diff_matches(
|
||||||
|
BenchmarkDiffer(style, None),
|
||||||
|
[
|
||||||
|
({'a': None}, {}, {}),
|
||||||
|
({}, {'a': None}, {}),
|
||||||
|
],
|
||||||
|
nest_result=False,
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue
Block a user