Merge pull request #9372 from a3d4/improve-error-coverage-cmdlinetests

Cover cmdlineTests.sh errors

scripts/error_codes.py

@@ -127,6 +127,12 @@ def find_ids_in_test_files(file_names):
     return used_ids
 
 
+def find_ids_in_cmdline_test_err(file_name):
+    source = read_file(file_name)
+    pattern = r' \(\d\d\d\d\):'
+    return {m.group(0)[-6:-2] for m in re.finditer(pattern, source, flags=re.MULTILINE)}
+
+
 def print_ids(ids):
     for k, id in enumerate(sorted(ids)):
         if k % 10 > 0:
@@ -149,6 +155,11 @@ def examine_id_coverage(top_dir, used_ids):
     )
     covered_ids = find_ids_in_test_files(test_file_names)
 
+    # special case, we are interested in warnings which are ignored by regular tests:
+    # Warning (1878): SPDX license identifier not provided in source file. ....
+    # Warning (3420): Source file does not specify required compiler version!
+    covered_ids |= find_ids_in_cmdline_test_err(path.join(top_dir, "test", "cmdlineTests", "error_codes", "err"))
+
     print(f"IDs in source files: {len(used_ids)}")
     print(f"IDs in test files  : {len(covered_ids)} ({len(covered_ids) - len(used_ids)})")
     print()

test/cmdlineTests/error_codes/err

@@ -1,26 +1,29 @@
+Warning (1878): SPDX license identifier not provided in source file. Before publishing, consider adding a comment containing "SPDX-License-Identifier: <SPDX-License>" to each source file. Use "SPDX-License-Identifier: UNLICENSED" for non-open-source code. Please see https://spdx.org for more information.
+--> error_codes/input.sol
+
 Error (4937): No visibility specified. Did you intend to add "public"?
- --> error_codes/input.sol:4:5:
+ --> error_codes/input.sol:2:5:
   |
-4 |     function f() {
+2 |     function f() {
   |     ^ (Relevant source part starts here and spans across multiple lines).
 
 Warning (3420): Source file does not specify required compiler version!
 --> error_codes/input.sol
 
 Error (4247): Expression has to be an lvalue.
- --> error_codes/input.sol:5:9:
+ --> error_codes/input.sol:3:9:
   |
-5 |         2=0;
+3 |         2=0;
   |         ^
 
 Error (7407): Type int_const 0 is not implicitly convertible to expected type int_const 2.
- --> error_codes/input.sol:5:11:
+ --> error_codes/input.sol:3:11:
   |
-5 |         2=0;
+3 |         2=0;
   |           ^
 
 Error (2614): Indexed expression has to be a type, mapping or array (is literal_string "")
- --> error_codes/input.sol:6:9:
+ --> error_codes/input.sol:4:9:
   |
-6 |         ""[2];
+4 |         ""[2];
   |         ^^

test/cmdlineTests/error_codes/input.sol

@@ -1,5 +1,3 @@
-// SPDX-License-Identifier: GPL-3.0
-
 contract C {
     function f() {
         2=0;

test/libsolidity/syntaxTests/imports/declaration_not_found.sol

@@ -0,0 +1,7 @@
+==== Source: a ====
+contract A { }
+==== Source: b ====
+import {C} from "a";
+contract B { }
+// ----
+// DeclarationError 2904: (b:0-20): Declaration "C" not found in "a" (referenced as "a").

test/libsolidity/syntaxTests/unexpected.sol

@@ -0,0 +1,3 @@
+unexpected
+// ----
+// ParserError 7858: (0-10): Expected pragma, import directive or contract/interface/library/struct/enum definition.