Add check that regular and unicode string literals are well formatted

liblangutil/Scanner.cpp

@@ -509,7 +509,7 @@ void Scanner::scanToken()
 		{
 		case '"':
 		case '\'':
-			token = scanString();
+			token = scanString(false);
 			break;
 		case '<':
 			// < <= << <<=
@@ -684,6 +684,18 @@ void Scanner::scanToken()
 					else
 						token = setError(ScannerError::IllegalToken);
 				}
+				else if (token == Token::Unicode)
+				{
+					// reset
+					m = 0;
+					n = 0;
+
+					// Special quoted hex string must follow
+					if (m_char == '"' || m_char == '\'')
+						token = scanString(true);
+					else
+						token = setError(ScannerError::IllegalToken);
+				}
 			}
 			else if (isDecimalDigit(m_char))
 				token = scanNumber();
@@ -775,7 +787,7 @@ bool Scanner::isUnicodeLinebreak()
 	return false;
 }
 
-Token Scanner::scanString()
+Token Scanner::scanString(bool const _isUnicode)
 {
 	char const quote = m_char;
 	advance();  // consume quote
@@ -791,11 +803,13 @@ Token Scanner::scanString()
 		}
 		else
 		{
-			// Report error on non-printable characters in string literals.
+			// Report error on non-printable characters in string literals, however
+			// allow anything for unicode string literals, because their validity will
+			// be verified later (in the syntax checker).
 			//
 			// We are using a manual range and not isprint() to avoid
 			// any potential complications with locale.
-			if (static_cast<unsigned>(c) <= 0x1f || static_cast<unsigned>(c) >= 0x7f)
+			if (!_isUnicode && (static_cast<unsigned>(c) <= 0x1f || static_cast<unsigned>(c) >= 0x7f))
 				return setError(ScannerError::IllegalCharacterInString);
 			addLiteralChar(c);
 		}
@@ -804,7 +818,7 @@ Token Scanner::scanString()
 		return setError(ScannerError::IllegalStringEndQuote);
 	literal.complete();
 	advance();  // consume quote
-	return Token::StringLiteral;
+	return _isUnicode ? Token::UnicodeStringLiteral : Token::StringLiteral;
 }
 
 Token Scanner::scanHexString()