Merge pull request #9412 from ethereum/unicode-string

[BREAKING] Support unicode string literal type

libsolidity/analysis/SyntaxChecker.cpp

@@ -28,6 +28,8 @@
 #include <liblangutil/ErrorReporter.h>
 #include <liblangutil/SemVerHandler.h>
 
+#include <libsolutil/UTF8.h>
+
 #include <boost/algorithm/string.hpp>
 
 #include <memory>
@@ -37,7 +39,7 @@ using namespace std;
 using namespace solidity;
 using namespace solidity::langutil;
 using namespace solidity::frontend;
-
+using namespace solidity::util;
 
 bool SyntaxChecker::checkSyntax(ASTNode const& _astRoot)
 {
@@ -217,6 +219,13 @@ bool SyntaxChecker::visit(Throw const& _throwStatement)
 
 bool SyntaxChecker::visit(Literal const& _literal)
 {
+	if ((_literal.token() == Token::UnicodeStringLiteral) && !validateUTF8(_literal.value()))
+		m_errorReporter.syntaxError(
+			8452_error,
+			_literal.location(),
+			"Invalid UTF-8 sequence found"
+		);
+
 	if (_literal.token() != Token::Number)
 		return true;
 

libsolidity/ast/ASTJsonConverter.cpp

@@ -920,6 +920,8 @@ string ASTJsonConverter::literalTokenKind(Token _token)
 		return "number";
 	case Token::StringLiteral:
 		return "string";
+	case Token::UnicodeStringLiteral:
+		return "unicodeString";
 	case Token::HexStringLiteral:
 		return "hexString";
 	case Token::TrueLiteral:

libsolidity/ast/ASTJsonImporter.cpp

@@ -943,6 +943,8 @@ Token ASTJsonImporter::literalTokenKind(Json::Value const& _node)
 		tok = Token::Number;
 	else if (_node["kind"].asString() == "string")
 		tok = Token::StringLiteral;
+	else if (_node["kind"].asString() == "unicodeString")
+		tok = Token::UnicodeStringLiteral;
 	else if (_node["kind"].asString() == "hexString")
 		tok = Token::HexStringLiteral;
 	else if (_node["kind"].asString() == "bool")

libsolidity/ast/TypeProvider.cpp

@@ -349,6 +349,7 @@ TypePointer TypeProvider::forLiteral(Literal const& _literal)
 	case Token::Number:
 		return rationalNumber(_literal);
 	case Token::StringLiteral:
+	case Token::UnicodeStringLiteral:
 	case Token::HexStringLiteral:
 		return stringLiteral(_literal.value());
 	default:

libsolidity/ast/Types.cpp

@@ -1408,7 +1408,7 @@ BoolResult StringLiteralType::isImplicitlyConvertibleTo(Type const& _convertTo)
 		return
 			arrayType->isByteArray() &&
 			!(arrayType->dataStoredIn(DataLocation::Storage) && arrayType->isPointer()) &&
-			!(arrayType->isString() && !isValidUTF8());
+			!(arrayType->isString() && !util::validateUTF8(value()));
 	else
 		return false;
 }
@@ -1442,11 +1442,6 @@ TypePointer StringLiteralType::mobileType() const
 	return TypeProvider::stringMemory();
 }
 
-bool StringLiteralType::isValidUTF8() const
-{
-	return util::validateUTF8(m_value);
-}
-
 FixedBytesType::FixedBytesType(unsigned _bytes): m_bytes(_bytes)
 {
 	solAssert(

libsolidity/ast/Types.h

@@ -629,8 +629,6 @@ public:
 	std::string toString(bool) const override;
 	TypePointer mobileType() const override;
 
-	bool isValidUTF8() const;
-
 	std::string const& value() const { return m_value; }
 
 protected:

libsolidity/parsing/Parser.cpp

@@ -1782,6 +1782,7 @@ ASTPointer<Expression> Parser::parsePrimaryExpression()
 		}
 		break;
 	case Token::StringLiteral:
+	case Token::UnicodeStringLiteral:
 	case Token::HexStringLiteral:
 	{
 		string literal = m_scanner->currentLiteral();