tests added and changes made

fixed some silly problems in Token.cpp

windows error fix
This commit is contained in:
RJ Catalano
2016-02-18 11:22:56 -06:00
parent 7b918a7bc7
commit f4da126018
7 changed files with 222 additions and 184 deletions
+11 -20
View File
@@ -82,11 +82,14 @@ bool isWhiteSpace(char c)
{
return c == ' ' || c == '\n' || c == '\t' || c == '\r';
}
bool isIdentifierPart(char c)
bool isIdentifierStart(char c)
{
return c == '_' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
}
bool isIdentifierPart(char c)
{
return isIdentifierStart(c) || isDecimalDigit(c);
}
int hexValue(char c)
{
if (c >= '0' && c <= '9')
@@ -383,7 +386,7 @@ void Scanner::scanToken()
m_nextSkippedComment.extendedTokenInfo.clear();
Token::Value token;
string tokenExtension = "";
string tokenExtension;
do
{
// Remember the position of the next token
@@ -550,7 +553,7 @@ void Scanner::scanToken()
token = selectToken(Token::BitNot);
break;
default:
if (isIdentifierPart(m_char))
if (isIdentifierStart(m_char))
tie(token, tokenExtension) = scanIdentifierOrKeyword();
else if (isDecimalDigit(m_char))
token = scanNumber();
@@ -700,7 +703,7 @@ Token::Value Scanner::scanNumber(char _charSeen)
// not be an identifier start or a decimal digit; see ECMA-262
// section 7.8.3, page 17 (note that we read only one decimal digit
// if the value is 0).
if (isDecimalDigit(m_char) || isIdentifierPart(m_char))
if (isDecimalDigit(m_char) || isIdentifierStart(m_char))
return Token::Illegal;
literal.complete();
return Token::Number;
@@ -708,26 +711,14 @@ Token::Value Scanner::scanNumber(char _charSeen)
tuple<Token::Value, string> Scanner::scanIdentifierOrKeyword()
{
solAssert(isIdentifierPart(m_char), "");
solAssert(isIdentifierStart(m_char), "");
LiteralScope literal(this, LITERAL_TYPE_STRING);
addLiteralCharAndAdvance();
// Scan the rest of the identifier characters.
string keyword = "";
string description = "";
while (isIdentifierPart(m_char)) //get main keyword
addLiteralCharAndAdvance();
keyword = m_nextToken.literal;
while (isDecimalDigit(m_char) || isIdentifierPart(m_char)) //get the description
while (isIdentifierPart(m_char)) //get full literal
addLiteralCharAndAdvance();
literal.complete();
if (m_nextToken.literal.find_first_of("0123456789") != string::npos)
{
description = m_nextToken.literal.substr(m_nextToken.literal.find_first_of("0123456789"));
keyword += "M";
if (description.find('x') != string::npos)
keyword += "xN";
}
return make_tuple(Token::fromIdentifierOrKeyword(keyword), description);
return Token::fromIdentifierOrKeyword(m_nextToken.literal);
}
char CharStream::advanceAndGet(size_t _chars)