adapt to latest code changes

liblangutil/Scanner.cpp

@@ -311,7 +311,7 @@ Token Scanner::skipMultiLineComment()
 		}
 	}
 	// Unterminated multi-line comment.
-	return Token::Illegal;
+	return Token::IllegalCommentTerminator;
 }
 
 Token Scanner::scanMultiLineDocComment()
@@ -362,7 +362,7 @@ Token Scanner::scanMultiLineDocComment()
 	}
 	literal.complete();
 	if (!endFound)
-		return Token::Illegal;
+		return Token::IllegalCommentTerminator;
 	else
 		return Token::CommentLiteral;
 }
@@ -392,7 +392,7 @@ Token Scanner::scanSlash()
 	{
 		// doxygen style /** natspec comment
 		if (!advance()) /* slash star comment before EOS */
-			return Token::Illegal;
+			return Token::IllegalCommentTerminator;
 		else if (m_char == '*')
 		{
 			advance(); //consume the last '*' at /**
@@ -409,8 +409,9 @@ Token Scanner::scanSlash()
 			comment = scanMultiLineDocComment();
 			m_nextSkippedComment.location.end = sourcePos();
 			m_nextSkippedComment.token = comment;
-			if (comment == Token::Illegal)
+			// @todo possibly: if (comment.isIllegal) return comment; to pass all errors
-				return Token::Illegal;
+			if (comment == Token::IllegalCommentTerminator)
+				return Token::IllegalCommentTerminator;
 			else
 				return Token::Whitespace;
 		}
@@ -620,6 +621,7 @@ void Scanner::scanToken()
 			else if (isSourcePastEndOfInput())
 				token = Token::EOS;
 			else
+				// @todo verfiy if this is actually an "IllegalUnknown" case
 				token = selectToken(Token::Illegal);
 			break;
 		}
@@ -713,13 +715,13 @@ Token Scanner::scanString()
 		if (c == '\\')
 		{
 			if (isSourcePastEndOfInput() || !scanEscape())
-				return Token::Illegal;
+				return Token::IllegalStringEscape;
 		}
 		else
 			addLiteralChar(c);
 	}
 	if (m_char != quote)
-		return Token::Illegal;
+		return Token::IllegalStringEndQuote;
 	literal.complete();
 	advance();  // consume quote
 	return Token::StringLiteral;
@@ -767,7 +769,8 @@ Token Scanner::scanNumber(char _charSeen)
 		// we have already seen a decimal point of the float
 		addLiteralChar('.');
 		if (m_char == '_')
-			return Token::Illegal;
+			// @todo add test-case (change of return value did not break test)
+			return Token::IllegalNumberSeparator;
 		scanDecimalDigits();  // we know we have at least one digit
 	}
 	else
@@ -784,14 +787,14 @@ Token Scanner::scanNumber(char _charSeen)
 				kind = HEX;
 				addLiteralCharAndAdvance();
 				if (!isHexDigit(m_char))
-					return Token::Illegal; // we must have at least one hex digit after 'x'
+					return Token::IllegalHexDigit; // we must have at least one hex digit after 'x'
 
 				while (isHexDigit(m_char) || m_char == '_') // We keep the underscores for later validation
 					addLiteralCharAndAdvance();
 			}
 			else if (isDecimalDigit(m_char))
 				// We do not allow octal numbers
-				return Token::Illegal;
+				return Token::IllegalOctalNotAllowed;
 		}
 		// Parse decimal digits and allow trailing fractional part.
 		if (kind == DECIMAL)
@@ -823,7 +826,8 @@ Token Scanner::scanNumber(char _charSeen)
 	{
 		solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
 		if (kind != DECIMAL)
-			return Token::Illegal;
+			// @todo add test (change introduced no failing)
+			return Token::IllegalExponent;
 		else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
 		{
 			// Recover from wrongly placed underscore as delimiter in literal with scientific
@@ -839,7 +843,7 @@ Token Scanner::scanNumber(char _charSeen)
 		if (m_char == '+' || m_char == '-')
 			addLiteralCharAndAdvance();
 		if (!isDecimalDigit(m_char))
-			return Token::Illegal; // we must have at least one decimal digit after 'e'/'E'
+			return Token::IllegalExponent; // we must have at least one decimal digit after 'e'/'E'
 		scanDecimalDigits();
 	}
 	// The source character immediately following a numeric literal must
@@ -847,7 +851,7 @@ Token Scanner::scanNumber(char _charSeen)
 	// section 7.8.3, page 17 (note that we read only one decimal digit
 	// if the value is 0).
 	if (isDecimalDigit(m_char) || isIdentifierStart(m_char))
-		return Token::Illegal;
+		return Token::IllegalNumberEnd;
 	literal.complete();
 	return Token::Number;
 }

liblangutil/Token.h

@@ -265,6 +265,15 @@ namespace langutil
 	T(Illegal, "ILLEGAL", 0)                                           \
 	/* Illegal hex token */                                            \
 	T(IllegalHex, "ILLEGAL_HEX", 0)                                    \
+	T(IllegalCommentTerminator, "ILLEGAL_COMMENT_TERMINATOR", 0)       \
+	T(IllegalStringEscape, "ILLEGAL_STRING_ESCAPE", 0)                 \
+	T(IllegalStringEndQuote, "ILLEGAL_STRING_END_QUOTE", 0)            \
+	T(IllegalNumberSeparator, "ILLEGAL_NUMER_SEPARATOR", 0)            \
+	T(IllegalHexDigit, "ILLEGAL_HEX_DIGIT", 0)                         \
+	T(IllegalOctalNotAllowed, "ILLEGAL_OCTAL_NOT_ALLOWED", 0)          \
+	T(IllegalExponent, "ILLEGAL_EXPONENT", 0)                          \
+	T(IllegalNumberEnd, "ILLEGAL_NUMBER_END", 0)                       \
+	T(IllegalEnd, NULL, 0) /* used as type Illegal enum end marker */  \
 	\
 	/* Scanner-internal use only. */                                   \
 	T(Whitespace, nullptr, 0)
@@ -312,6 +321,9 @@ namespace TokenTraits
 	constexpr bool isTimeSubdenomination(Token op) { return op == Token::SubSecond || op == Token::SubMinute || op == Token::SubHour || op == Token::SubDay || op == Token::SubWeek || op == Token::SubYear; }
 	constexpr bool isReservedKeyword(Token op) { return (Token::Abstract <= op && op <= Token::Unchecked); }
 
+	// @returns true if token is illegal
+	constexpr bool isIllegal(Token tok) { return Token::Illegal <= tok && tok < Token::IllegalEnd; };
+
 	inline Token AssignmentToBinaryOp(Token op)
 	{
 		solAssert(isAssignmentOp(op) && op != Token::Assign, "");

libsolidity/parsing/Parser.cpp

@@ -1558,6 +1558,30 @@ ASTPointer<Expression> Parser::parsePrimaryExpression()
 	case Token::IllegalHex:
 		fatalParserError("Expected even number of hex-nibbles within double-quotes.");
 		break;
+	case Token::IllegalCommentTerminator:
+		fatalParserError("Expected multi-line comment-terminator.");
+		break;
+	case Token::IllegalStringEscape:
+		fatalParserError("Invalid String Escape.");
+		break;
+	case Token::IllegalStringEndQuote:
+		fatalParserError("Expected String end-quote.");
+		break;
+	case Token::IllegalNumberSeparator:
+		fatalParserError("Invalid use of Number Separator '_'.");
+		break;
+	case Token::IllegalHexDigit:
+		fatalParserError("Hex Digit missing or invalid.");
+		break;
+	case Token::IllegalOctalNotAllowed:
+		fatalParserError("Octal Numbers not allowed.");
+		break;
+	case Token::IllegalExponent:
+		fatalParserError("Invalid Exponent.");
+		break;
+	case Token::IllegalNumberEnd:
+		fatalParserError("Digit or Identifier-Start not allowed at end of Number.");
+		break;
 	default:
 		if (TokenTraits::isElementaryTypeName(token))
 		{

test/libsolidity/SolidityScanner.cpp

@@ -59,6 +59,8 @@ BOOST_AUTO_TEST_CASE(smoke_test)
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
+// STRING ESCAPES
+
 BOOST_AUTO_TEST_CASE(string_escapes)
 {
 	Scanner scanner(CharStream("  { \"a\\x61\""));
@@ -87,12 +89,12 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal)
 {
 	Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)"));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEscape);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
 	// TODO recovery from illegal tokens should be improved
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalNumberEnd);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEndQuote);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
@@ -110,18 +112,19 @@ BOOST_AUTO_TEST_CASE(hex_numbers)
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234");
 	scanner.reset(CharStream("0X1234"), "");
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	// @todo update underlying code to return "Expected lower-case 'x' for Hex-Numbers." or similar.
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalNumberEnd);
 }
 
 BOOST_AUTO_TEST_CASE(octal_numbers)
 {
 	Scanner scanner(CharStream("07"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalOctalNotAllowed);
 	scanner.reset(CharStream("007"), "");
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalOctalNotAllowed);
 	scanner.reset(CharStream("-07"), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalOctalNotAllowed);
 	scanner.reset(CharStream("-.07"), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
@@ -204,7 +207,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
 BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
 {
 	Scanner scanner(CharStream("0x_abc"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalHexDigit);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -469,7 +472,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
 {
 	Scanner scanner(CharStream("{ \"\\uFFnicode\""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
-	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEscape);
 }
 
 //  HEX STRING LITERAL
@@ -516,7 +519,7 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
 {
 	// This used to parse as "comment", "identifier"
 	Scanner scanner(CharStream("/** / x"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalCommentTerminator);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
@@ -524,14 +527,14 @@ BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos)
 {
 	// This used to parse as "whitespace"
 	Scanner scanner(CharStream("/**"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalCommentTerminator);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(multiline_comment_at_eos)
 {
 	Scanner scanner(CharStream("/*"));
-	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalCommentTerminator);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
@@ -594,10 +597,10 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings)
 	for (auto const& nl: {"\n", "\r"})
 	{
 		Scanner scanner(CharStream("\"abc " + string(nl) + " def\""));
-		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalStringEndQuote);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
-		BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+		BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEndQuote);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 	}
 }
@@ -607,12 +610,12 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings)
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
 		Scanner scanner(CharStream("\"abc " + string(nl) + " def\""));
-		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
+		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalStringEndQuote);
 		for (size_t i = 0; i < string(nl).size(); i++)
 			BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
-		BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
+		BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEndQuote);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 	}
 }

test/libsolidity/syntaxTests/inlineAssembly/invalid/invalid_number.sol

@@ -7,4 +7,4 @@ contract C {
 }
 // ----
 // ParserError: (72-73): Literal, identifier or instruction expected.
-// ParserError: (72-73): Expected primary expression.
+// ParserError: (72-73): Octal Numbers not allowed.

test/libsolidity/syntaxTests/parsing/invalid_fixed_conversion_leading_zeroes_check.sol

@@ -4,4 +4,4 @@ contract test {
 	}
 }
 // ----
-// ParserError: (44-47): Expected primary expression.
+// ParserError: (44-47): Digit or Identifier-Start not allowed at end of Number.

test/libsolidity/syntaxTests/string/string_new_line.sol

@@ -6,4 +6,4 @@ contract test {
     }
 }
 // ----
-// ParserError: (100-112): Expected primary expression.
+// ParserError: (100-112): Expected String end-quote.

test/libsolidity/syntaxTests/string/string_terminated_by_backslash.sol

@@ -5,4 +5,4 @@ contract test {
     }
 }
 // ----
-// ParserError: (100-109): Expected primary expression.
+// ParserError: (100-109): Expected String end-quote.

test/libsolidity/syntaxTests/string/string_unterminated.sol

@@ -4,4 +4,4 @@ contract test {
     }
 }
 // ----
-// ParserError: (100-112): Expected primary expression.
+// ParserError: (100-112): Expected String end-quote.

test/libsolidity/syntaxTests/string/string_unterminated_no_new_line.sol

@@ -1,4 +1,4 @@
 contract test {
     function f() pure public { "abc\
 // ----
-// ParserError: (47-53): Expected primary expression.
+// ParserError: (47-53): Expected String end-quote.

test/libsolidity/syntaxTests/unicode_escape_literals.sol

@@ -28,4 +28,4 @@ contract test {
 
 }
 // ----
-// ParserError: (678-681): Expected primary expression.
+// ParserError: (678-681): Invalid String Escape.