Cleanup of scanner.

Removed redundancy of keyword definitions and removed some unused token predicates.
2023-10-03 13:03:40 +00:00 · 2014-12-01 13:46:04 +01:00 · 2014-12-01 13:46:04 +01:00 · b685735b3a
commit b685735b3a
parent 3fc2708d65
2 changed files with 11 additions and 198 deletions
--- a/Scanner.cpp
+++ b/Scanner.cpp
@ -194,7 +194,6 @@ Token::Value Scanner::selectToken(char _next, Token::Value _then, Token::Value _
 		return _else;
 }

-
 bool Scanner::skipWhitespace()
 {
 	int const startPosition = getSourcePos();
@ -204,7 +203,6 @@ bool Scanner::skipWhitespace()
 	return getSourcePos() != startPosition;
 }

-
 Token::Value Scanner::skipSingleLineComment()
 {
 	// The line terminator at the end of the line is not considered
@ -215,7 +213,6 @@ Token::Value Scanner::skipSingleLineComment()
 	return Token::WHITESPACE;
 }

-/// For the moment this function simply consumes a single line triple slash doc comment
 Token::Value Scanner::scanDocumentationComment()
 {
 	LiteralScope literal(this, LITERAL_TYPE_COMMENT);
@ -545,14 +542,12 @@ Token::Value Scanner::scanString()
 	return Token::STRING_LITERAL;
 }

-
 void Scanner::scanDecimalDigits()
 {
 	while (isDecimalDigit(m_char))
 		addLiteralCharAndAdvance();
 }

-
 Token::Value Scanner::scanNumber(char _charSeen)
 {
 	enum { DECIMAL, HEX, BINARY } kind = DECIMAL;
@ -623,186 +618,18 @@ Token::Value Scanner::scanNumber(char _charSeen)
 // ----------------------------------------------------------------------------
 // Keyword Matcher

-#define KEYWORDS(KEYWORD_GROUP, KEYWORD)                                       \
-	KEYWORD_GROUP('a')                                                         \
-	KEYWORD("address", Token::ADDRESS)                                         \
-	KEYWORD_GROUP('b')                                                         \
-	KEYWORD("break", Token::BREAK)                                             \
-	KEYWORD("bool", Token::BOOL)                                               \
-	KEYWORD_GROUP('c')                                                         \
-	KEYWORD("case", Token::CASE)                                               \
-	KEYWORD("const", Token::CONST)                                             \
-	KEYWORD("continue", Token::CONTINUE)                                       \
-	KEYWORD("contract", Token::CONTRACT)                                       \
-	KEYWORD_GROUP('d')                                                         \
-	KEYWORD("default", Token::DEFAULT)                                         \
-	KEYWORD("delete", Token::DELETE)                                           \
-	KEYWORD("do", Token::DO)                                                   \
-	KEYWORD_GROUP('e')                                                         \
-	KEYWORD("else", Token::ELSE)                                               \
-	KEYWORD("extends", Token::EXTENDS)                                         \
-	KEYWORD_GROUP('f')                                                         \
-	KEYWORD("false", Token::FALSE_LITERAL)                                     \
-	KEYWORD("for", Token::FOR)                                                 \
-	KEYWORD("function", Token::FUNCTION)                                       \
-	KEYWORD_GROUP('h')                                                         \
-	KEYWORD("hash", Token::HASH)                                               \
-	KEYWORD("hash8", Token::HASH8)                                             \
-	KEYWORD("hash16", Token::HASH16)                                           \
-	KEYWORD("hash24", Token::HASH24)                                           \
-	KEYWORD("hash32", Token::HASH32)                                           \
-	KEYWORD("hash40", Token::HASH40)                                           \
-	KEYWORD("hash48", Token::HASH48)                                           \
-	KEYWORD("hash56", Token::HASH56)                                           \
-	KEYWORD("hash64", Token::HASH64)                                           \
-	KEYWORD("hash72", Token::HASH72)                                           \
-	KEYWORD("hash80", Token::HASH80)                                           \
-	KEYWORD("hash88", Token::HASH88)                                           \
-	KEYWORD("hash96", Token::HASH96)                                           \
-	KEYWORD("hash104", Token::HASH104)                                         \
-	KEYWORD("hash112", Token::HASH112)                                         \
-	KEYWORD("hash120", Token::HASH120)                                         \
-	KEYWORD("hash128", Token::HASH128)                                         \
-	KEYWORD("hash136", Token::HASH136)                                         \
-	KEYWORD("hash144", Token::HASH144)                                         \
-	KEYWORD("hash152", Token::HASH152)                                         \
-	KEYWORD("hash160", Token::HASH160)                                         \
-	KEYWORD("hash168", Token::HASH168)                                         \
-	KEYWORD("hash178", Token::HASH176)                                         \
-	KEYWORD("hash184", Token::HASH184)                                         \
-	KEYWORD("hash192", Token::HASH192)                                         \
-	KEYWORD("hash200", Token::HASH200)                                         \
-	KEYWORD("hash208", Token::HASH208)                                         \
-	KEYWORD("hash216", Token::HASH216)                                         \
-	KEYWORD("hash224", Token::HASH224)                                         \
-	KEYWORD("hash232", Token::HASH232)                                         \
-	KEYWORD("hash240", Token::HASH240)                                         \
-	KEYWORD("hash248", Token::HASH248)                                         \
-	KEYWORD("hash256", Token::HASH256)                                         \
-	KEYWORD_GROUP('i')                                                         \
-	KEYWORD("if", Token::IF)                                                   \
-	KEYWORD("in", Token::IN)                                                   \
-	KEYWORD("int", Token::INT)                                                 \
-	KEYWORD("int8", Token::INT8)                                               \
-	KEYWORD("int16", Token::INT16)                                             \
-	KEYWORD("int24", Token::INT24)                                             \
-	KEYWORD("int32", Token::INT32)                                             \
-	KEYWORD("int40", Token::INT40)                                             \
-	KEYWORD("int48", Token::INT48)                                             \
-	KEYWORD("int56", Token::INT56)                                             \
-	KEYWORD("int64", Token::INT64)                                             \
-	KEYWORD("int72", Token::INT72)                                             \
-	KEYWORD("int80", Token::INT80)                                             \
-	KEYWORD("int88", Token::INT88)                                             \
-	KEYWORD("int96", Token::INT96)                                             \
-	KEYWORD("int104", Token::INT104)                                           \
-	KEYWORD("int112", Token::INT112)                                           \
-	KEYWORD("int120", Token::INT120)                                           \
-	KEYWORD("int128", Token::INT128)                                           \
-	KEYWORD("int136", Token::INT136)                                           \
-	KEYWORD("int144", Token::INT144)                                           \
-	KEYWORD("int152", Token::INT152)                                           \
-	KEYWORD("int160", Token::INT160)                                           \
-	KEYWORD("int168", Token::INT168)                                           \
-	KEYWORD("int178", Token::INT176)                                           \
-	KEYWORD("int184", Token::INT184)                                           \
-	KEYWORD("int192", Token::INT192)                                           \
-	KEYWORD("int200", Token::INT200)                                           \
-	KEYWORD("int208", Token::INT208)                                           \
-	KEYWORD("int216", Token::INT216)                                           \
-	KEYWORD("int224", Token::INT224)                                           \
-	KEYWORD("int232", Token::INT232)                                           \
-	KEYWORD("int240", Token::INT240)                                           \
-	KEYWORD("int248", Token::INT248)                                           \
-	KEYWORD("int256", Token::INT256)                                           \
-	KEYWORD_GROUP('l')                                                         \
-	KEYWORD_GROUP('m')                                                         \
-	KEYWORD("mapping", Token::MAPPING)                                         \
-	KEYWORD_GROUP('n')                                                         \
-	KEYWORD("new", Token::NEW)                                                 \
-	KEYWORD("null", Token::NULL_LITERAL)                                       \
-	KEYWORD_GROUP('p')                                                         \
-	KEYWORD("private", Token::PRIVATE)                                         \
-	KEYWORD("public", Token::PUBLIC)                                           \
-	KEYWORD_GROUP('r')                                                         \
-	KEYWORD("real", Token::REAL)                                               \
-	KEYWORD("return", Token::RETURN)                                           \
-	KEYWORD("returns", Token::RETURNS)                                         \
-	KEYWORD_GROUP('s')                                                         \
-	KEYWORD("string", Token::STRING_TYPE)                                      \
-	KEYWORD("struct", Token::STRUCT)                                           \
-	KEYWORD("switch", Token::SWITCH)                                           \
-	KEYWORD_GROUP('t')                                                         \
-	KEYWORD("text", Token::TEXT)                                               \
-	KEYWORD("true", Token::TRUE_LITERAL)                                       \
-	KEYWORD_GROUP('u')                                                         \
-	KEYWORD("uint", Token::UINT)                                               \
-	KEYWORD("uint8", Token::UINT8)                                             \
-	KEYWORD("uint16", Token::UINT16)                                           \
-	KEYWORD("uint24", Token::UINT24)                                           \
-	KEYWORD("uint32", Token::UINT32)                                           \
-	KEYWORD("uint40", Token::UINT40)                                           \
-	KEYWORD("uint48", Token::UINT48)                                           \
-	KEYWORD("uint56", Token::UINT56)                                           \
-	KEYWORD("uint64", Token::UINT64)                                           \
-	KEYWORD("uint72", Token::UINT72)                                           \
-	KEYWORD("uint80", Token::UINT80)                                           \
-	KEYWORD("uint88", Token::UINT88)                                           \
-	KEYWORD("uint96", Token::UINT96)                                           \
-	KEYWORD("uint104", Token::UINT104)                                         \
-	KEYWORD("uint112", Token::UINT112)                                         \
-	KEYWORD("uint120", Token::UINT120)                                         \
-	KEYWORD("uint128", Token::UINT128)                                         \
-	KEYWORD("uint136", Token::UINT136)                                         \
-	KEYWORD("uint144", Token::UINT144)                                         \
-	KEYWORD("uint152", Token::UINT152)                                         \
-	KEYWORD("uint160", Token::UINT160)                                         \
-	KEYWORD("uint168", Token::UINT168)                                         \
-	KEYWORD("uint178", Token::UINT176)                                         \
-	KEYWORD("uint184", Token::UINT184)                                         \
-	KEYWORD("uint192", Token::UINT192)                                         \
-	KEYWORD("uint200", Token::UINT200)                                         \
-	KEYWORD("uint208", Token::UINT208)                                         \
-	KEYWORD("uint216", Token::UINT216)                                         \
-	KEYWORD("uint224", Token::UINT224)                                         \
-	KEYWORD("uint232", Token::UINT232)                                         \
-	KEYWORD("uint240", Token::UINT240)                                         \
-	KEYWORD("uint248", Token::UINT248)                                         \
-	KEYWORD("uint256", Token::UINT256)                                         \
-	KEYWORD("ureal", Token::UREAL)                                             \
-	KEYWORD_GROUP('v')                                                         \
-	KEYWORD("var", Token::VAR)                                                 \
-	KEYWORD_GROUP('w')                                                         \
-	KEYWORD("while", Token::WHILE)                                             \

-
-static Token::Value KeywordOrIdentifierToken(string const& _input)
+static Token::Value keywordOrIdentifierToken(string const& _input)
 {
-	if (asserts(!_input.empty()))
-		BOOST_THROW_EXCEPTION(InternalCompilerError());
-	int const kMinLength = 2;
-	int const kMaxLength = 10;
-	if (_input.size() < kMinLength || _input.size() > kMaxLength)
-		return Token::IDENTIFIER;
-	switch (_input[0])
-	{
-	default:
-#define KEYWORD_GROUP_CASE(ch)					\
-		break;									\
-	case ch:
-#define KEYWORD(keyword, token)                                         \
-		{                                                               \
-			/* 'keyword' is a char array, so sizeof(keyword) is */      \
-			/* strlen(keyword) plus 1 for the NUL char. */              \
-			int const keywordLength = sizeof(keyword) - 1;              \
-			BOOST_STATIC_ASSERT(keywordLength >= kMinLength);           \
-			BOOST_STATIC_ASSERT(keywordLength <= kMaxLength);           \
-			if (_input == keyword)                                     \
-				return token;                                           \
-	}
-		KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD)
-	}
-	return Token::IDENTIFIER;
+#define KEYWORD(name, string, precedence) {string, Token::name},
+#define TOKEN(name, string, precedence)
+	static const map<string, Token::Value> keywords{
+		TOKEN_LIST(TOKEN, KEYWORD)
+	};
+#undef KEYWORD
+#undef TOKEN
+	auto it = keywords.find(_input);
+	return it == keywords.end() ? Token::IDENTIFIER : it->second;
 }

 Token::Value Scanner::scanIdentifierOrKeyword()
@ -815,7 +642,7 @@ Token::Value Scanner::scanIdentifierOrKeyword()
 	while (isIdentifierPart(m_char))
 		addLiteralCharAndAdvance();
 	literal.complete();
-	return KeywordOrIdentifierToken(m_nextToken.literal);
+	return keywordOrIdentifierToken(m_nextToken.literal);
 }

 char CharStream::advanceAndGet(size_t _chars)
--- a/Token.h
+++ b/Token.h
@ -314,25 +314,11 @@ public:
 	}

 	// Predicates
-	static bool isKeyword(Value tok) { return m_tokenType[tok] == 'K'; }
-	static bool isIdentifier(Value tok) { return tok == IDENTIFIER; }
 	static bool isElementaryTypeName(Value tok) { return INT <= tok && tok < TYPES_END; }
 	static bool isAssignmentOp(Value tok) { return ASSIGN <= tok && tok <= ASSIGN_MOD; }
 	static bool isBinaryOp(Value op) { return COMMA <= op && op <= MOD; }
-	static bool isTruncatingBinaryOp(Value op) { return BIT_OR <= op && op <= SHR; }
 	static bool isArithmeticOp(Value op) { return ADD <= op && op <= MOD; }
 	static bool isCompareOp(Value op) { return EQ <= op && op <= IN; }
-	static bool isOrderedRelationalCompareOp(Value op)
-	{
-		return op == LT || op == LTE || op == GT || op == GTE;
-	}
-	static bool isEqualityOp(Value op) { return op == EQ; }
-	static bool isInequalityOp(Value op) { return op == NE; }
-	static bool isArithmeticCompareOp(Value op)
-	{
-		return isOrderedRelationalCompareOp(op) ||
-			   isEqualityOp(op) || isInequalityOp(op);
-	}

 	static Value AssignmentToBinaryOp(Value op)
 	{