From 9a5db2dfed65aaeb72e16d706a6be325848de171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 22 Dec 2020 04:10:58 +0100 Subject: [PATCH] prepare_report.py: Print some statistics about contracts and errors --- scripts/bytecodecompare/prepare_report.py | 67 ++++++++++++++----- .../test_bytecodecompare_prepare_report.py | 54 ++++++++++++++- 2 files changed, 104 insertions(+), 17 deletions(-) diff --git a/scripts/bytecodecompare/prepare_report.py b/scripts/bytecodecompare/prepare_report.py index 2dd738022..8d62f47fa 100755 --- a/scripts/bytecodecompare/prepare_report.py +++ b/scripts/bytecodecompare/prepare_report.py @@ -80,6 +80,33 @@ class FileReport: return '.' +@dataclass +class Statistics: + file_count: int = 0 + contract_count: int = 0 + error_count: int = 0 + missing_bytecode_count: int = 0 + missing_metadata_count: int = 0 + + def aggregate(self, report: FileReport): + contract_reports = report.contract_reports if report.contract_reports is not None else [] + + self.file_count += 1 + self.contract_count += len(contract_reports) + self.error_count += (1 if report.contract_reports is None else 0) + self.missing_bytecode_count += sum(1 for c in contract_reports if c.bytecode is None) + self.missing_metadata_count += sum(1 for c in contract_reports if c.metadata is None) + + def __str__(self) -> str: + return "test cases: {}, contracts: {}, errors: {}, missing bytecode: {}, missing metadata: {}".format( + self.file_count, + str(self.contract_count) + ('+' if self.error_count > 0 else ''), + self.error_count, + self.missing_bytecode_count, + self.missing_metadata_count, + ) + + def load_source(path: Union[Path, str], smt_use: SMTUse) -> str: with open(path, mode='r', encoding='utf8') as source_file: file_content = source_file.read() @@ -225,22 +252,30 @@ def run_compiler(compiler_path: Path, source_file_name: Path, optimize: bool, fo def generate_report(source_file_names: List[str], compiler_path: Path, interface: CompilerInterface, smt_use: SMTUse, force_no_optimize_yul: bool, report_file_path: Path, verbose: bool, exit_on_error: bool): - with open(report_file_path, mode='w', encoding='utf8', newline='\n') as report_file: - for optimize in [False, True]: - with TemporaryDirectory(prefix='prepare_report-') as tmp_dir: - for source_file_name in sorted(source_file_names): - try: - report = run_compiler(Path(compiler_path), Path(source_file_name), optimize, force_no_optimize_yul, interface, smt_use, Path(tmp_dir), exit_on_error) - print(report.format_summary(verbose), end=('\n' if verbose else ''), flush=True) - report_file.write(report.format_report()) - except subprocess.CalledProcessError as exception: - print(f"\n\nInterrupted by an exception while processing file '{source_file_name}' with optimize={optimize}\n", file=sys.stderr) - print(f"COMPILER STDOUT:\n{exception.stdout}", file=sys.stderr) - print(f"COMPILER STDERR:\n{exception.stderr}", file=sys.stderr) - raise - except: - print(f"\n\nInterrupted by an exception while processing file '{source_file_name}' with optimize={optimize}\n", file=sys.stderr) - raise + statistics = Statistics() + + try: + with open(report_file_path, mode='w', encoding='utf8', newline='\n') as report_file: + for optimize in [False, True]: + with TemporaryDirectory(prefix='prepare_report-') as tmp_dir: + for source_file_name in sorted(source_file_names): + try: + report = run_compiler(Path(compiler_path), Path(source_file_name), optimize, force_no_optimize_yul, interface, smt_use, Path(tmp_dir), exit_on_error) + + statistics.aggregate(report) + print(report.format_summary(verbose), end=('\n' if verbose else ''), flush=True) + + report_file.write(report.format_report()) + except subprocess.CalledProcessError as exception: + print(f"\n\nInterrupted by an exception while processing file '{source_file_name}' with optimize={optimize}\n", file=sys.stderr) + print(f"COMPILER STDOUT:\n{exception.stdout}", file=sys.stderr) + print(f"COMPILER STDERR:\n{exception.stderr}", file=sys.stderr) + raise + except: + print(f"\n\nInterrupted by an exception while processing file '{source_file_name}' with optimize={optimize}\n", file=sys.stderr) + raise + finally: + print('\n', statistics, '\n', sep='') def commandline_parser() -> ArgumentParser: diff --git a/test/scripts/test_bytecodecompare_prepare_report.py b/test/scripts/test_bytecodecompare_prepare_report.py index 725169156..7121fb677 100644 --- a/test/scripts/test_bytecodecompare_prepare_report.py +++ b/test/scripts/test_bytecodecompare_prepare_report.py @@ -7,7 +7,7 @@ from textwrap import dedent from unittest_helpers import LIBSOLIDITY_TEST_DIR, load_fixture, load_libsolidity_test_case -from bytecodecompare.prepare_report import CompilerInterface, FileReport, ContractReport, SMTUse +from bytecodecompare.prepare_report import CompilerInterface, FileReport, ContractReport, SMTUse, Statistics from bytecodecompare.prepare_report import load_source, parse_cli_output, parse_standard_json_output, prepare_compiler_input @@ -87,6 +87,58 @@ class TestPrepareReport_FileReport(unittest.TestCase): self.assertEqual(report.format_report(), '') +class TestPrepareReport_Statistics(unittest.TestCase): + def test_initialization(self): + self.assertEqual(Statistics(), Statistics(0, 0, 0, 0, 0)) + + def test_aggregate_bytecode_and_metadata_present(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ContractReport('C', 'c.sol', 'B', 'M')])) + self.assertEqual(statistics, Statistics(1, 1, 0, 0, 0)) + + def test_aggregate_bytecode_missing(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ContractReport('C', 'c.sol', None, 'M')])) + self.assertEqual(statistics, Statistics(1, 1, 0, 1, 0)) + + def test_aggregate_metadata_missing(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ContractReport('C', 'c.sol', 'B', None)])) + self.assertEqual(statistics, Statistics(1, 1, 0, 0, 1)) + + def test_aggregate_no_contract_reports(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[])) + self.assertEqual(statistics, Statistics(1, 0, 0, 0, 0)) + + def test_aggregate_missing_contract_report_list(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=None)) + self.assertEqual(statistics, Statistics(1, 0, 1, 0, 0)) + + def test_aggregate_multiple_contract_reports(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ + ContractReport('C', 'c.sol', 'B', 'M'), + ContractReport('C', 'c.sol', None, 'M'), + ContractReport('C', 'c.sol', 'B', None), + ContractReport('C', 'c.sol', None, None), + ])) + self.assertEqual(statistics, Statistics(1, 4, 0, 2, 2)) + + def test_str(self): + statistics = Statistics() + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=[ + ContractReport('C', 'c.sol', 'B', 'M'), + ContractReport('C', 'c.sol', None, 'M'), + ContractReport('C', 'c.sol', 'B', None), + ContractReport('C', 'c.sol', None, None), + ])) + statistics.aggregate(FileReport(file_name=Path('F'), contract_reports=None)) + + self.assertEqual(statistics, Statistics(2, 4, 1, 2, 2)) + self.assertEqual(str(statistics), "test cases: 2, contracts: 4+, errors: 1, missing bytecode: 2, missing metadata: 2") + class TestPrepareReport(unittest.TestCase): def setUp(self):