From 0c3e0a65cfd6435e9ba0c7f0768db9804743b6e2 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Mon, 3 Aug 2020 04:28:35 +0200 Subject: [PATCH] Extend error_codes.py to to detect newly introduced error codes, not covered by tests --- scripts/error_codes.py | 63 ++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/scripts/error_codes.py b/scripts/error_codes.py index eaf401cb0..6e9159b74 100755 --- a/scripts/error_codes.py +++ b/scripts/error_codes.py @@ -167,7 +167,7 @@ def print_ids_per_file(ids, id_to_file_names, top_dir): print() -def examine_id_coverage(top_dir, source_id_to_file_names): +def examine_id_coverage(top_dir, source_id_to_file_names, new_ids_only=False): test_sub_dirs = [ path.join("test", "libsolidity", "errorRecoveryTests"), path.join("test", "libsolidity", "smtCheckerTests"), @@ -196,24 +196,51 @@ def examine_id_coverage(top_dir, source_id_to_file_names): assert len(test_ids & white_ids) == 0, "The sets are not supposed to intersect" test_ids |= white_ids - print(f"IDs in source files: {len(source_ids)}") - print(f"IDs in test files : {len(test_ids)} ({len(test_ids) - len(source_ids)})") - print() - test_only_ids = test_ids - source_ids - if len(test_only_ids) != 0: - print("Error. The following error codes found in tests, but not in sources:") - print_ids(test_only_ids) - return 1 - source_only_ids = source_ids - test_ids - if len(source_only_ids) != 0: - print("The following error codes found in sources, but not in tests:") - print_ids_per_file(source_only_ids, source_id_to_file_names, top_dir) - print("\n\nPlease make sure to add appropriate tests.") - return 1 - return 0 + if not new_ids_only: + print(f"IDs in source files: {len(source_ids)}") + print(f"IDs in test files : {len(test_ids)} ({len(test_ids) - len(source_ids)})") + print() + + if len(test_only_ids) != 0: + print("Error. The following error codes found in tests, but not in sources:") + print_ids(test_only_ids) + return False + + if len(source_only_ids) != 0: + print("The following error codes found in sources, but not in tests:") + print_ids_per_file(source_only_ids, source_id_to_file_names, top_dir) + print("\n\nPlease make sure to add appropriate tests.") + return False + + old_source_only_ids = { + "1054", "1123", "1133", "1220", "1395", "1584", "1733", "1823", "1950", "1957", + "1988", "2418", "2461", "2512", "2592", "2657", "2800", "2842", "2856", "3069", + "3263", "3287", "3299", "3356", "3384", "3441", "3517", "3682", "3781", "3876", + "3893", "3947", "3997", "4010", "4110", "4294", "4802", "4805", "4828", "4846", + "4904", "4990", "5052", "5073", "5170", "5188", "5272", "5333", "5347", "5473", + "5622", "6041", "6052", "6272", "6708", "6792", "6931", "7110", "7128", "7186", + "7319", "7589", "7593", "7653", "7771", "7812", "7885", "8065", "8084", "8140", + "8143", "8261", "8311", "8312", "8452", "8592", "8758", "8794", "9005", "9011", + "9085", "9102", "9114", "9390", "9439", "9440", "9547", "9551", "9615", "9980" + } + new_source_only_ids = source_only_ids - old_source_only_ids + if len(new_source_only_ids) != 0: + print("The following new error code(s), not covered by tests, found:") + print_ids(new_source_only_ids) + print( + "\nYou can:\n" + "- create appropriate test(s);\n" + "- add the error code(s) to old_source_only_ids in error_codes.py\n" + " (to silence the checking script, with a promise to add a test later);\n" + "- add the error code(s) to white_ids in error_codes.py\n" + " (for rare cases when the error is not supposed to be tested)" + ) + return False + + return True def main(argv): @@ -267,9 +294,11 @@ def main(argv): if not ok: print("Incorrect IDs have to be fixed before applying --examine-coverage") exit(1) - res = examine_id_coverage(cwd, source_id_to_file_names) + res = 0 if examine_id_coverage(cwd, source_id_to_file_names) else 1 exit(res) + ok &= examine_id_coverage(cwd, source_id_to_file_names, new_ids_only=True) + random.seed() if next_id: