Cleanup & polish numbers-with-underscores parsing, also improving tests.

test/libsolidity/SolidityEndToEndTest.cpp

@@ -12840,7 +12840,7 @@ BOOST_AUTO_TEST_CASE(test_underscore_in_hex)
 {
 	char const* sourceCode = R"(
 		contract test {
-			function f(bool cond) returns (uint) {
+			function f(bool cond) public pure returns (uint) {
 				uint32 x = 0x1234_ab;
 				uint y = 0x1234_abcd_1234;
 				return cond ? x : y;

test/libsolidity/SolidityScanner.cpp

@@ -155,170 +155,74 @@ BOOST_AUTO_TEST_CASE(trailing_dot)
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
-BOOST_AUTO_TEST_CASE(underscores_in_integer)
-{
-	Scanner scanner(CharStream("var x = 1_23_4;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
-	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1234");
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-}
-
-BOOST_AUTO_TEST_CASE(underscores_in_scientific_notation)
-{
-	Scanner scanner(CharStream("var x = 1_2e10;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
-	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "12e10");
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-}
-
-BOOST_AUTO_TEST_CASE(underscores_in_scientific_notation_in_exp_part)
-{
-	Scanner scanner(CharStream("var x = 12e1_0;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
-	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "12e10");
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-}
-
-
-BOOST_AUTO_TEST_CASE(underscores_in_hex)
-{
-	Scanner scanner(CharStream("var x = 0xab_19cf;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
-	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0xab_19cf");
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-}
-
-BOOST_AUTO_TEST_CASE(leading_underscore_integer_is_identifier)
-{
-	Scanner scanner(CharStream("var x = _12;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-}
-
 BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
 {
-	Scanner scanner(CharStream("var x = _1.2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
+	// Actual error is cought by SyntaxChecker.
+	Scanner scanner(CharStream("_1.2"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
 {
-	Scanner scanner(CharStream("var x = 1._2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	// Actual error is cought by SyntaxChecker.
+	Scanner scanner(CharStream("1._2"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
+
+	scanner.reset(CharStream("1._"), "");
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
 {
-	Scanner scanner(CharStream("var x = _1e2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
+	// Actual error is cought by SyntaxChecker.
+	Scanner scanner(CharStream("_1e2"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
 {
-	Scanner scanner(CharStream("var x = 1e_2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	// Actual error is cought by SyntaxChecker.
+	Scanner scanner(CharStream("1e_2"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2");
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
 {
-	Scanner scanner(CharStream("var x = 0x_abc;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
+	Scanner scanner(CharStream("0x_abc"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
-BOOST_AUTO_TEST_CASE(trailing_underscore_integer_illegal)
+BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
 {
-	Scanner scanner(CharStream("var x = 12_;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	// Actual error is cought by SyntaxChecker.
+	Scanner scanner(CharStream("12._1234_1234"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
-BOOST_AUTO_TEST_CASE(leading_underscore_after_decimal_illegal)
+BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos)
 {
-	Scanner scanner(CharStream("var x = 1.2_;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-}
+	// Actual error is cought by SyntaxChecker.
+	Scanner scanner(CharStream("0x123_"));
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-BOOST_AUTO_TEST_CASE(leading_underscore_before_decimal_illegal)
-{
-	Scanner scanner(CharStream("var x = 1_.2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-}
+	scanner.reset(CharStream("123_"), "");
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-BOOST_AUTO_TEST_CASE(trailing_underscore_exp_illegal)
-{
-	Scanner scanner(CharStream("var x = 1e2_;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-}
-
-BOOST_AUTO_TEST_CASE(trailing_underscore_exp_before_e_illegal)
-{
-	Scanner scanner(CharStream("var x = 1_e2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-}
-
-BOOST_AUTO_TEST_CASE(trailing_underscore_hex_illegal)
-{
-	Scanner scanner(CharStream("var x = 0xabc_;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-}
-
-BOOST_AUTO_TEST_CASE(double_underscore_illegal)
-{
-	Scanner scanner(CharStream("var x = 1__2;"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	scanner.reset(CharStream("12.34_"), "");
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(negative_numbers)

test/libsolidity/syntaxTests/parsing/lexer_numbers_with_underscores_decimal.sol

@@ -0,0 +1,13 @@
+contract C {
+  function f() public pure {
+    uint d1 = 654_321;
+    uint d2 =  54_321;
+    uint d3 =   4_321;
+    uint d4 = 5_43_21;
+    uint d5 = 1_2e10;
+    uint d6 = 12e1_0;
+
+    d1; d2; d3; d4; d5; d6;
+  }
+}
+// ----

test/libsolidity/syntaxTests/parsing/lexer_numbers_with_underscores_decimal_fail.sol

@@ -0,0 +1,13 @@
+contract C {
+  function f() public pure {
+    uint D1 = 1234_;
+    uint D2 = 12__34;
+    uint D3 = 12_e34;
+    uint D4 = 12e_34;
+  }
+}
+// ----
+// SyntaxError: (56-61): Invalid use of underscores in number literal. No trailing underscores allowed.
+// SyntaxError: (77-83): Invalid use of underscores in number literal. Only one consecutive underscores between digits allowed.
+// SyntaxError: (99-105): Invalid use of underscores in number literal. No underscore at the end of the mantissa allowed.
+// SyntaxError: (121-127): Invalid use of underscores in number literal. No underscore in front of exponent allowed.

test/libsolidity/syntaxTests/parsing/lexer_numbers_with_underscores_fixed.sol

@@ -0,0 +1,9 @@
+contract C {
+  function f() public pure {
+    fixed f1 = 3.14_15;
+    fixed f2 = 3_1.4_15;
+
+    f1; f2;
+  }
+}
+// ----

test/libsolidity/syntaxTests/parsing/lexer_numbers_with_underscores_fixed_fail.sol

@@ -0,0 +1,17 @@
+contract C {
+  function f() public pure {
+    fixed F1 = 3.1415_;
+    fixed F2 = 3__1.4__15;
+    fixed F3 = 1_.2;
+    fixed F4 = 1._2;
+    fixed F5 = 1.2e_12;
+    fixed F6 = 1._;
+  }
+}
+// ----
+// SyntaxError: (57-64): Invalid use of underscores in number literal. No trailing underscores allowed.
+// SyntaxError: (81-91): Invalid use of underscores in number literal. Only one consecutive underscores between digits allowed.
+// SyntaxError: (108-112): Invalid use of underscores in number literal. No underscores in front of the fraction part allowed.
+// SyntaxError: (129-133): Invalid use of underscores in number literal. No underscores in front of the fraction part allowed.
+// SyntaxError: (150-157): Invalid use of underscores in number literal. No underscore in front of exponent allowed.
+// SyntaxError: (174-177): Invalid use of underscores in number literal. No trailing underscores allowed.

test/libsolidity/syntaxTests/parsing/lexer_numbers_with_underscores_hex.sol

@@ -0,0 +1,13 @@
+contract C {
+  function f() public pure {
+    uint x1 = 0x8765_4321;
+    uint x2 = 0x765_4321;
+    uint x3 = 0x65_4321;
+    uint x4 = 0x5_4321;
+    uint x5 = 0x123_1234_1234_1234;
+    uint x6 = 0x123456_1234_1234;
+
+    x1; x2; x3; x4; x5; x6;
+  }
+}
+// ----

test/libsolidity/syntaxTests/parsing/lexer_numbers_with_underscores_hex_fail.sol

@@ -0,0 +1,7 @@
+contract C {
+  function f() public pure {
+    uint X1 = 0x1234__1234__1234__123;
+  }
+}
+// ----
+// SyntaxError: (56-79): Invalid use of underscores in number literal. Only one consecutive underscores between digits allowed.