2014-10-20 14:37:04 +00:00
|
|
|
/*
|
2016-11-18 23:13:20 +00:00
|
|
|
This file is part of solidity.
|
2014-10-20 14:37:04 +00:00
|
|
|
|
2016-11-18 23:13:20 +00:00
|
|
|
solidity is free software: you can redistribute it and/or modify
|
2014-10-20 14:37:04 +00:00
|
|
|
it under the terms of the GNU General Public License as published by
|
|
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
|
|
(at your option) any later version.
|
|
|
|
|
2016-11-18 23:13:20 +00:00
|
|
|
solidity is distributed in the hope that it will be useful,
|
2014-10-20 14:37:04 +00:00
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
2016-11-18 23:13:20 +00:00
|
|
|
along with solidity. If not, see <http://www.gnu.org/licenses/>.
|
2014-10-20 14:37:04 +00:00
|
|
|
|
|
|
|
This file is derived from the file "scanner.cc", which was part of the
|
|
|
|
V8 project. The original copyright header follows:
|
|
|
|
|
|
|
|
Copyright 2006-2012, the V8 project authors. All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
|
|
modification, are permitted provided that the following conditions are
|
|
|
|
met:
|
|
|
|
|
|
|
|
* Redistributions of source code must retain the above copyright
|
|
|
|
notice, this list of conditions and the following disclaimer.
|
|
|
|
* Redistributions in binary form must reproduce the above
|
|
|
|
copyright notice, this list of conditions and the following
|
|
|
|
disclaimer in the documentation and/or other materials provided
|
|
|
|
with the distribution.
|
|
|
|
* Neither the name of Google Inc. nor the names of its
|
|
|
|
contributors may be used to endorse or promote products derived
|
|
|
|
from this software without specific prior written permission.
|
|
|
|
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
/**
|
|
|
|
* @author Christian <c@ethdev.com>
|
|
|
|
* @date 2014
|
|
|
|
* Solidity scanner.
|
|
|
|
*/
|
2014-10-06 15:13:52 +00:00
|
|
|
|
2014-10-08 18:53:50 +00:00
|
|
|
#include <algorithm>
|
|
|
|
#include <tuple>
|
2017-06-21 20:50:57 +00:00
|
|
|
#include <libsolidity/interface/Exceptions.h>
|
2015-10-20 22:21:52 +00:00
|
|
|
#include <libsolidity/parsing/Scanner.h>
|
2014-10-06 15:13:52 +00:00
|
|
|
|
2014-10-24 17:06:30 +00:00
|
|
|
using namespace std;
|
|
|
|
|
2014-10-16 12:08:54 +00:00
|
|
|
namespace dev
|
|
|
|
{
|
|
|
|
namespace solidity
|
|
|
|
{
|
2014-10-06 15:13:52 +00:00
|
|
|
|
2014-10-16 12:08:54 +00:00
|
|
|
namespace
|
|
|
|
{
|
2014-11-21 16:08:35 +00:00
|
|
|
bool isDecimalDigit(char c)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
|
|
|
return '0' <= c && c <= '9';
|
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
bool isHexDigit(char c)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
2014-11-21 16:08:35 +00:00
|
|
|
return isDecimalDigit(c)
|
2014-10-16 12:08:54 +00:00
|
|
|
|| ('a' <= c && c <= 'f')
|
|
|
|
|| ('A' <= c && c <= 'F');
|
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
bool isLineTerminator(char c)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
|
|
|
return c == '\n';
|
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
bool isWhiteSpace(char c)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
2014-12-18 13:43:35 +00:00
|
|
|
return c == ' ' || c == '\n' || c == '\t' || c == '\r';
|
2014-10-16 12:08:54 +00:00
|
|
|
}
|
2016-02-09 21:43:23 +00:00
|
|
|
bool isIdentifierStart(char c)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
2016-02-08 21:43:22 +00:00
|
|
|
return c == '_' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
|
2014-10-16 12:08:54 +00:00
|
|
|
}
|
2016-02-09 21:43:23 +00:00
|
|
|
bool isIdentifierPart(char c)
|
|
|
|
{
|
|
|
|
return isIdentifierStart(c) || isDecimalDigit(c);
|
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
int hexValue(char c)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
2014-10-20 12:00:37 +00:00
|
|
|
if (c >= '0' && c <= '9')
|
|
|
|
return c - '0';
|
|
|
|
else if (c >= 'a' && c <= 'f')
|
|
|
|
return c - 'a' + 10;
|
|
|
|
else if (c >= 'A' && c <= 'F')
|
|
|
|
return c - 'A' + 10;
|
2014-10-16 12:08:54 +00:00
|
|
|
else return -1;
|
|
|
|
}
|
2014-10-20 12:00:37 +00:00
|
|
|
} // end anonymous namespace
|
2014-10-06 15:13:52 +00:00
|
|
|
|
2014-11-30 22:25:42 +00:00
|
|
|
|
|
|
|
|
|
|
|
/// Scoped helper for literal recording. Automatically drops the literal
|
|
|
|
/// if aborting the scanning before it's complete.
|
|
|
|
enum LiteralType {
|
|
|
|
LITERAL_TYPE_STRING,
|
|
|
|
LITERAL_TYPE_NUMBER, // not really different from string type in behaviour
|
|
|
|
LITERAL_TYPE_COMMENT
|
|
|
|
};
|
|
|
|
|
|
|
|
class LiteralScope
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
explicit LiteralScope(Scanner* _self, enum LiteralType _type): m_type(_type)
|
|
|
|
, m_scanner(_self)
|
|
|
|
, m_complete(false)
|
|
|
|
{
|
|
|
|
if (_type == LITERAL_TYPE_COMMENT)
|
|
|
|
m_scanner->m_nextSkippedComment.literal.clear();
|
|
|
|
else
|
|
|
|
m_scanner->m_nextToken.literal.clear();
|
|
|
|
}
|
|
|
|
~LiteralScope()
|
|
|
|
{
|
|
|
|
if (!m_complete)
|
|
|
|
{
|
|
|
|
if (m_type == LITERAL_TYPE_COMMENT)
|
|
|
|
m_scanner->m_nextSkippedComment.literal.clear();
|
|
|
|
else
|
|
|
|
m_scanner->m_nextToken.literal.clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
void complete() { m_complete = true; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
enum LiteralType m_type;
|
|
|
|
Scanner* m_scanner;
|
|
|
|
bool m_complete;
|
|
|
|
}; // end of LiteralScope class
|
|
|
|
|
|
|
|
|
2014-12-03 17:52:28 +00:00
|
|
|
void Scanner::reset(CharStream const& _source, string const& _sourceName)
|
2014-10-06 15:13:52 +00:00
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
m_source = _source;
|
2014-12-03 17:52:28 +00:00
|
|
|
m_sourceName = make_shared<string const>(_sourceName);
|
2014-12-03 16:45:12 +00:00
|
|
|
reset();
|
|
|
|
}
|
|
|
|
|
|
|
|
void Scanner::reset()
|
|
|
|
{
|
|
|
|
m_source.reset();
|
2014-10-09 10:28:37 +00:00
|
|
|
m_char = m_source.get();
|
|
|
|
skipWhitespace();
|
2014-11-30 21:43:40 +00:00
|
|
|
scanToken();
|
2014-12-01 00:05:55 +00:00
|
|
|
next();
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2014-11-05 13:20:56 +00:00
|
|
|
bool Scanner::scanHexByte(char& o_scannedByte)
|
2014-10-06 15:13:52 +00:00
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
char x = 0;
|
2014-11-05 13:20:56 +00:00
|
|
|
for (int i = 0; i < 2; i++)
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
2014-11-21 16:08:35 +00:00
|
|
|
int d = hexValue(m_char);
|
2014-10-16 12:08:54 +00:00
|
|
|
if (d < 0)
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
rollback(i);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
x = x * 16 + d;
|
|
|
|
advance();
|
|
|
|
}
|
2014-11-05 13:20:56 +00:00
|
|
|
o_scannedByte = x;
|
2014-10-09 10:28:37 +00:00
|
|
|
return true;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2016-06-07 18:23:19 +00:00
|
|
|
bool Scanner::scanUnicode(unsigned & o_codepoint)
|
|
|
|
{
|
|
|
|
unsigned x = 0;
|
|
|
|
for (int i = 0; i < 4; i++)
|
|
|
|
{
|
|
|
|
int d = hexValue(m_char);
|
|
|
|
if (d < 0)
|
|
|
|
{
|
|
|
|
rollback(i);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
x = x * 16 + d;
|
|
|
|
advance();
|
|
|
|
}
|
|
|
|
o_codepoint = x;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// This supports codepoints between 0000 and FFFF.
|
2016-08-01 13:10:46 +00:00
|
|
|
void Scanner::addUnicodeAsUTF8(unsigned codepoint)
|
2016-06-07 18:23:19 +00:00
|
|
|
{
|
|
|
|
if (codepoint <= 0x7f)
|
|
|
|
addLiteralChar(codepoint);
|
|
|
|
else if (codepoint <= 0x7ff)
|
|
|
|
{
|
|
|
|
addLiteralChar(0xc0 | (codepoint >> 6));
|
|
|
|
addLiteralChar(0x80 | (codepoint & 0x3f));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
addLiteralChar(0xe0 | (codepoint >> 12));
|
|
|
|
addLiteralChar(0x80 | ((codepoint >> 6) & 0x3f));
|
|
|
|
addLiteralChar(0x80 | (codepoint & 0x3f));
|
|
|
|
}
|
|
|
|
}
|
2014-10-06 15:13:52 +00:00
|
|
|
|
|
|
|
// Ensure that tokens can be stored in a byte.
|
|
|
|
BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
|
|
|
|
|
2014-11-20 22:56:24 +00:00
|
|
|
Token::Value Scanner::next()
|
2014-10-06 15:13:52 +00:00
|
|
|
{
|
2014-11-21 16:08:35 +00:00
|
|
|
m_currentToken = m_nextToken;
|
2014-11-30 21:43:40 +00:00
|
|
|
m_skippedComment = m_nextSkippedComment;
|
|
|
|
scanToken();
|
|
|
|
|
2014-11-21 16:08:35 +00:00
|
|
|
return m_currentToken.token;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2014-10-16 21:49:45 +00:00
|
|
|
Token::Value Scanner::selectToken(char _next, Token::Value _then, Token::Value _else)
|
|
|
|
{
|
|
|
|
advance();
|
|
|
|
if (m_char == _next)
|
2014-10-17 10:52:39 +00:00
|
|
|
return selectToken(_then);
|
2014-10-16 21:49:45 +00:00
|
|
|
else
|
|
|
|
return _else;
|
|
|
|
}
|
|
|
|
|
2014-10-06 15:13:52 +00:00
|
|
|
bool Scanner::skipWhitespace()
|
|
|
|
{
|
2015-08-31 16:44:29 +00:00
|
|
|
int const startPosition = sourcePos();
|
2014-11-21 16:08:35 +00:00
|
|
|
while (isWhiteSpace(m_char))
|
2014-10-16 12:08:54 +00:00
|
|
|
advance();
|
2014-10-09 10:28:37 +00:00
|
|
|
// Return whether or not we skipped any characters.
|
2015-08-31 16:44:29 +00:00
|
|
|
return sourcePos() != startPosition;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2018-09-06 09:05:35 +00:00
|
|
|
void Scanner::skipWhitespaceExceptUnicodeLinebreak()
|
2014-12-18 12:27:25 +00:00
|
|
|
{
|
2018-09-06 09:05:35 +00:00
|
|
|
while (isWhiteSpace(m_char) && !isUnicodeLinebreak())
|
2014-12-18 12:27:25 +00:00
|
|
|
advance();
|
|
|
|
}
|
|
|
|
|
2014-10-06 15:13:52 +00:00
|
|
|
Token::Value Scanner::skipSingleLineComment()
|
|
|
|
{
|
2018-09-06 09:05:35 +00:00
|
|
|
// Line terminator is not part of the comment. If it is a
|
|
|
|
// non-ascii line terminator, it will result in a parser error.
|
|
|
|
while (!isUnicodeLinebreak())
|
2015-09-10 12:26:34 +00:00
|
|
|
if (!advance()) break;
|
|
|
|
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Whitespace;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2014-12-17 17:53:18 +00:00
|
|
|
Token::Value Scanner::scanSingleLineDocComment()
|
2014-11-18 17:50:40 +00:00
|
|
|
{
|
2014-11-27 17:57:50 +00:00
|
|
|
LiteralScope literal(this, LITERAL_TYPE_COMMENT);
|
2014-12-18 12:27:25 +00:00
|
|
|
advance(); //consume the last '/' at ///
|
2018-09-06 09:05:35 +00:00
|
|
|
|
|
|
|
skipWhitespaceExceptUnicodeLinebreak();
|
|
|
|
|
2014-11-27 17:57:50 +00:00
|
|
|
while (!isSourcePastEndOfInput())
|
2014-11-18 17:50:40 +00:00
|
|
|
{
|
2014-11-27 17:57:50 +00:00
|
|
|
if (isLineTerminator(m_char))
|
|
|
|
{
|
|
|
|
// check if next line is also a documentation comment
|
|
|
|
skipWhitespace();
|
2014-11-30 21:43:40 +00:00
|
|
|
if (!m_source.isPastEndOfInput(3) &&
|
|
|
|
m_source.get(0) == '/' &&
|
2014-11-27 17:57:50 +00:00
|
|
|
m_source.get(1) == '/' &&
|
2014-11-30 21:43:40 +00:00
|
|
|
m_source.get(2) == '/')
|
2014-11-27 17:57:50 +00:00
|
|
|
{
|
|
|
|
addCommentLiteralChar('\n');
|
2014-11-30 21:43:40 +00:00
|
|
|
m_char = m_source.advanceAndGet(3);
|
2014-11-27 17:57:50 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
break; // next line is not a documentation comment, we are done
|
|
|
|
|
|
|
|
}
|
2018-09-06 09:05:35 +00:00
|
|
|
else if (isUnicodeLinebreak())
|
|
|
|
// Any line terminator that is not '\n' is considered to end the
|
|
|
|
// comment.
|
|
|
|
break;
|
2014-11-21 08:09:39 +00:00
|
|
|
addCommentLiteralChar(m_char);
|
2014-11-18 17:50:40 +00:00
|
|
|
advance();
|
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
literal.complete();
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::CommentLiteral;
|
2014-11-18 17:50:40 +00:00
|
|
|
}
|
|
|
|
|
2014-10-06 15:13:52 +00:00
|
|
|
Token::Value Scanner::skipMultiLineComment()
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
advance();
|
2014-10-16 12:08:54 +00:00
|
|
|
while (!isSourcePastEndOfInput())
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
char ch = m_char;
|
|
|
|
advance();
|
2014-10-16 21:49:45 +00:00
|
|
|
|
2014-10-09 10:28:37 +00:00
|
|
|
// If we have reached the end of the multi-line comment, we
|
|
|
|
// consume the '/' and insert a whitespace. This way all
|
|
|
|
// multi-line comments are treated as whitespace.
|
2014-10-16 12:08:54 +00:00
|
|
|
if (ch == '*' && m_char == '/')
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
m_char = ' ';
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Whitespace;
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// Unterminated multi-line comment.
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2014-12-17 17:53:18 +00:00
|
|
|
Token::Value Scanner::scanMultiLineDocComment()
|
|
|
|
{
|
|
|
|
LiteralScope literal(this, LITERAL_TYPE_COMMENT);
|
|
|
|
bool endFound = false;
|
2014-12-18 12:27:25 +00:00
|
|
|
bool charsAdded = false;
|
2014-12-17 17:53:18 +00:00
|
|
|
|
2018-09-06 09:05:35 +00:00
|
|
|
while (isWhiteSpace(m_char) && !isLineTerminator(m_char))
|
|
|
|
advance();
|
|
|
|
|
2014-12-17 17:53:18 +00:00
|
|
|
while (!isSourcePastEndOfInput())
|
|
|
|
{
|
2014-12-18 12:27:25 +00:00
|
|
|
//handle newlines in multline comments
|
2014-12-17 17:53:18 +00:00
|
|
|
if (isLineTerminator(m_char))
|
|
|
|
{
|
|
|
|
skipWhitespace();
|
2016-11-30 16:28:07 +00:00
|
|
|
if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '*')
|
|
|
|
{ // it is unknown if this leads to the end of the comment
|
|
|
|
addCommentLiteralChar('*');
|
|
|
|
advance();
|
|
|
|
}
|
|
|
|
else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) != '/')
|
2014-12-18 12:27:25 +00:00
|
|
|
{ // skip first '*' in subsequent lines
|
|
|
|
if (charsAdded)
|
|
|
|
addCommentLiteralChar('\n');
|
|
|
|
m_char = m_source.advanceAndGet(2);
|
|
|
|
}
|
|
|
|
else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
|
|
|
|
{ // if after newline the comment ends, don't insert the newline
|
|
|
|
m_char = m_source.advanceAndGet(2);
|
|
|
|
endFound = true;
|
|
|
|
break;
|
2014-12-17 17:53:18 +00:00
|
|
|
}
|
2014-12-18 15:48:25 +00:00
|
|
|
else if (charsAdded)
|
2014-12-17 17:53:18 +00:00
|
|
|
addCommentLiteralChar('\n');
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
|
|
|
|
{
|
2014-12-18 12:27:25 +00:00
|
|
|
m_char = m_source.advanceAndGet(2);
|
2014-12-17 17:53:18 +00:00
|
|
|
endFound = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
addCommentLiteralChar(m_char);
|
2014-12-18 12:27:25 +00:00
|
|
|
charsAdded = true;
|
2014-12-17 17:53:18 +00:00
|
|
|
advance();
|
|
|
|
}
|
|
|
|
literal.complete();
|
|
|
|
if (!endFound)
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal;
|
2014-12-17 17:53:18 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::CommentLiteral;
|
2014-12-17 17:53:18 +00:00
|
|
|
}
|
|
|
|
|
2014-12-18 16:30:10 +00:00
|
|
|
Token::Value Scanner::scanSlash()
|
|
|
|
{
|
2015-08-31 16:44:29 +00:00
|
|
|
int firstSlashPosition = sourcePos();
|
2014-12-18 16:30:10 +00:00
|
|
|
advance();
|
|
|
|
if (m_char == '/')
|
|
|
|
{
|
|
|
|
if (!advance()) /* double slash comment directly before EOS */
|
2018-09-06 09:05:35 +00:00
|
|
|
return Token::Whitespace;
|
2014-12-18 16:30:10 +00:00
|
|
|
else if (m_char == '/')
|
|
|
|
{
|
|
|
|
// doxygen style /// comment
|
|
|
|
Token::Value comment;
|
|
|
|
m_nextSkippedComment.location.start = firstSlashPosition;
|
|
|
|
comment = scanSingleLineDocComment();
|
2015-08-31 16:44:29 +00:00
|
|
|
m_nextSkippedComment.location.end = sourcePos();
|
2014-12-18 16:30:10 +00:00
|
|
|
m_nextSkippedComment.token = comment;
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Whitespace;
|
2014-12-18 16:30:10 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
return skipSingleLineComment();
|
|
|
|
}
|
|
|
|
else if (m_char == '*')
|
|
|
|
{
|
|
|
|
// doxygen style /** natspec comment
|
|
|
|
if (!advance()) /* slash star comment before EOS */
|
2018-09-06 09:05:35 +00:00
|
|
|
return Token::Illegal;
|
2014-12-18 16:30:10 +00:00
|
|
|
else if (m_char == '*')
|
|
|
|
{
|
2015-01-05 15:37:43 +00:00
|
|
|
advance(); //consume the last '*' at /**
|
|
|
|
|
2018-09-06 09:05:35 +00:00
|
|
|
// "/**/"
|
|
|
|
if (m_char == '/')
|
2015-01-05 15:37:43 +00:00
|
|
|
{
|
2018-09-06 09:05:35 +00:00
|
|
|
advance(); //skip the closing slash
|
|
|
|
return Token::Whitespace;
|
2015-01-05 15:37:43 +00:00
|
|
|
}
|
2018-09-06 09:05:35 +00:00
|
|
|
// we actually have a multiline documentation comment
|
|
|
|
Token::Value comment;
|
|
|
|
m_nextSkippedComment.location.start = firstSlashPosition;
|
|
|
|
comment = scanMultiLineDocComment();
|
|
|
|
m_nextSkippedComment.location.end = sourcePos();
|
|
|
|
m_nextSkippedComment.token = comment;
|
|
|
|
if (comment == Token::Illegal)
|
|
|
|
return Token::Illegal;
|
|
|
|
else
|
|
|
|
return Token::Whitespace;
|
2014-12-18 16:30:10 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
return skipMultiLineComment();
|
|
|
|
}
|
|
|
|
else if (m_char == '=')
|
2015-02-09 13:00:12 +00:00
|
|
|
return selectToken(Token::AssignDiv);
|
2014-12-18 16:30:10 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Div;
|
2014-12-18 16:30:10 +00:00
|
|
|
}
|
|
|
|
|
2014-11-30 21:43:40 +00:00
|
|
|
void Scanner::scanToken()
|
2014-10-06 15:13:52 +00:00
|
|
|
{
|
2014-11-21 16:08:35 +00:00
|
|
|
m_nextToken.literal.clear();
|
2016-02-12 21:01:27 +00:00
|
|
|
m_nextToken.extendedTokenInfo = make_tuple(0, 0);
|
2014-11-30 21:43:40 +00:00
|
|
|
m_nextSkippedComment.literal.clear();
|
2016-02-12 21:01:27 +00:00
|
|
|
m_nextSkippedComment.extendedTokenInfo = make_tuple(0, 0);
|
2016-02-08 21:43:22 +00:00
|
|
|
|
2014-10-16 12:08:54 +00:00
|
|
|
Token::Value token;
|
2016-02-12 21:01:27 +00:00
|
|
|
// M and N are for the purposes of grabbing different type sizes
|
2016-02-15 16:34:45 +00:00
|
|
|
unsigned m;
|
|
|
|
unsigned n;
|
2014-10-16 12:08:54 +00:00
|
|
|
do
|
|
|
|
{
|
|
|
|
// Remember the position of the next token
|
2015-08-31 16:44:29 +00:00
|
|
|
m_nextToken.location.start = sourcePos();
|
2014-10-16 12:08:54 +00:00
|
|
|
switch (m_char)
|
|
|
|
{
|
|
|
|
case '"':
|
|
|
|
case '\'':
|
|
|
|
token = scanString();
|
|
|
|
break;
|
|
|
|
case '<':
|
|
|
|
// < <= << <<=
|
|
|
|
advance();
|
|
|
|
if (m_char == '=')
|
2015-02-10 08:52:19 +00:00
|
|
|
token = selectToken(Token::LessThanOrEqual);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '<')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken('=', Token::AssignShl, Token::SHL);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::LessThan;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '>':
|
|
|
|
// > >= >> >>= >>> >>>=
|
|
|
|
advance();
|
|
|
|
if (m_char == '=')
|
2015-02-10 08:52:19 +00:00
|
|
|
token = selectToken(Token::GreaterThanOrEqual);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '>')
|
|
|
|
{
|
|
|
|
// >> >>= >>> >>>=
|
|
|
|
advance();
|
|
|
|
if (m_char == '=')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::AssignSar);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '>')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken('=', Token::AssignShr, Token::SHR);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
|
|
|
token = Token::SAR;
|
|
|
|
}
|
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::GreaterThan;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '=':
|
|
|
|
// = == =>
|
|
|
|
advance();
|
|
|
|
if (m_char == '=')
|
2015-02-10 08:52:19 +00:00
|
|
|
token = selectToken(Token::Equal);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '>')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Arrow);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::Assign;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '!':
|
|
|
|
// ! !=
|
|
|
|
advance();
|
|
|
|
if (m_char == '=')
|
2015-02-10 08:52:19 +00:00
|
|
|
token = selectToken(Token::NotEqual);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::Not;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '+':
|
|
|
|
// + ++ +=
|
|
|
|
advance();
|
|
|
|
if (m_char == '+')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Inc);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '=')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::AssignAdd);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::Add;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '-':
|
2014-12-19 10:31:17 +00:00
|
|
|
// - -- -=
|
2014-10-16 12:08:54 +00:00
|
|
|
advance();
|
|
|
|
if (m_char == '-')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Dec);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '=')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::AssignSub);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::Sub;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '*':
|
2015-02-08 11:23:17 +00:00
|
|
|
// * ** *=
|
|
|
|
advance();
|
|
|
|
if (m_char == '*')
|
|
|
|
token = selectToken(Token::Exp);
|
|
|
|
else if (m_char == '=')
|
|
|
|
token = selectToken(Token::AssignMul);
|
|
|
|
else
|
|
|
|
token = Token::Mul;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '%':
|
|
|
|
// % %=
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken('=', Token::AssignMod, Token::Mod);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '/':
|
|
|
|
// / // /* /=
|
2014-12-18 16:30:10 +00:00
|
|
|
token = scanSlash();
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '&':
|
|
|
|
// & && &=
|
|
|
|
advance();
|
|
|
|
if (m_char == '&')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::And);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '=')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::AssignBitAnd);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::BitAnd;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '|':
|
|
|
|
// | || |=
|
|
|
|
advance();
|
|
|
|
if (m_char == '|')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Or);
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (m_char == '=')
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::AssignBitOr);
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::BitOr;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '^':
|
|
|
|
// ^ ^=
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken('=', Token::AssignBitXor, Token::BitXor);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '.':
|
|
|
|
// . Number
|
|
|
|
advance();
|
2014-11-21 16:08:35 +00:00
|
|
|
if (isDecimalDigit(m_char))
|
2014-11-05 07:40:21 +00:00
|
|
|
token = scanNumber('.');
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::Period;
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case ':':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Colon);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case ';':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Semicolon);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case ',':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Comma);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '(':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::LParen);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case ')':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::RParen);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '[':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::LBrack);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case ']':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::RBrack);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '{':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::LBrace);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '}':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::RBrace);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '?':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Conditional);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
case '~':
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::BitNot);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
|
|
|
default:
|
2016-02-09 21:43:23 +00:00
|
|
|
if (isIdentifierStart(m_char))
|
2016-08-16 14:31:23 +00:00
|
|
|
{
|
2016-02-15 16:34:45 +00:00
|
|
|
tie(token, m, n) = scanIdentifierOrKeyword();
|
2016-08-16 14:31:23 +00:00
|
|
|
|
|
|
|
// Special case for hexademical literals
|
|
|
|
if (token == Token::Hex)
|
|
|
|
{
|
|
|
|
// reset
|
|
|
|
m = 0;
|
|
|
|
n = 0;
|
|
|
|
|
|
|
|
// Special quoted hex string must follow
|
|
|
|
if (m_char == '"' || m_char == '\'')
|
|
|
|
token = scanHexString();
|
|
|
|
else
|
|
|
|
token = Token::Illegal;
|
|
|
|
}
|
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
else if (isDecimalDigit(m_char))
|
2014-11-05 07:40:21 +00:00
|
|
|
token = scanNumber();
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (skipWhitespace())
|
2015-02-09 13:00:12 +00:00
|
|
|
token = Token::Whitespace;
|
2014-10-16 12:08:54 +00:00
|
|
|
else if (isSourcePastEndOfInput())
|
|
|
|
token = Token::EOS;
|
|
|
|
else
|
2015-02-09 13:00:12 +00:00
|
|
|
token = selectToken(Token::Illegal);
|
2014-10-16 12:08:54 +00:00
|
|
|
break;
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
// Continue scanning for tokens as long as we're just skipping
|
|
|
|
// whitespace.
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2015-02-09 13:00:12 +00:00
|
|
|
while (token == Token::Whitespace);
|
2015-08-31 16:44:29 +00:00
|
|
|
m_nextToken.location.end = sourcePos();
|
2014-11-21 16:08:35 +00:00
|
|
|
m_nextToken.token = token;
|
2016-02-15 16:34:45 +00:00
|
|
|
m_nextToken.extendedTokenInfo = make_tuple(m, n);
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool Scanner::scanEscape()
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
char c = m_char;
|
|
|
|
advance();
|
|
|
|
// Skip escaped newlines.
|
2014-11-21 16:08:35 +00:00
|
|
|
if (isLineTerminator(c))
|
2014-10-09 10:28:37 +00:00
|
|
|
return true;
|
2014-10-16 12:08:54 +00:00
|
|
|
switch (c)
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
case '\'': // fall through
|
2014-10-16 21:49:45 +00:00
|
|
|
case '"': // fall through
|
2014-10-16 12:08:54 +00:00
|
|
|
case '\\':
|
|
|
|
break;
|
2014-10-16 21:49:45 +00:00
|
|
|
case 'b':
|
2014-10-16 12:08:54 +00:00
|
|
|
c = '\b';
|
|
|
|
break;
|
2014-10-16 21:49:45 +00:00
|
|
|
case 'f':
|
2014-10-16 12:08:54 +00:00
|
|
|
c = '\f';
|
|
|
|
break;
|
2014-10-16 21:49:45 +00:00
|
|
|
case 'n':
|
2014-10-16 12:08:54 +00:00
|
|
|
c = '\n';
|
|
|
|
break;
|
2014-10-16 21:49:45 +00:00
|
|
|
case 'r':
|
2014-10-16 12:08:54 +00:00
|
|
|
c = '\r';
|
|
|
|
break;
|
2014-10-16 21:49:45 +00:00
|
|
|
case 't':
|
2014-10-16 12:08:54 +00:00
|
|
|
c = '\t';
|
|
|
|
break;
|
2014-10-16 21:49:45 +00:00
|
|
|
case 'v':
|
2014-10-16 12:08:54 +00:00
|
|
|
c = '\v';
|
|
|
|
break;
|
2016-06-07 18:23:19 +00:00
|
|
|
case 'u':
|
|
|
|
{
|
|
|
|
unsigned codepoint;
|
|
|
|
if (!scanUnicode(codepoint))
|
|
|
|
return false;
|
2016-08-01 13:10:46 +00:00
|
|
|
addUnicodeAsUTF8(codepoint);
|
2016-06-07 18:23:19 +00:00
|
|
|
return true;
|
|
|
|
}
|
2014-10-16 21:49:45 +00:00
|
|
|
case 'x':
|
2014-11-05 13:20:56 +00:00
|
|
|
if (!scanHexByte(c))
|
2014-10-20 12:00:37 +00:00
|
|
|
return false;
|
2014-10-09 10:28:37 +00:00
|
|
|
break;
|
2018-09-06 09:05:35 +00:00
|
|
|
default:
|
|
|
|
return false;
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-10-16 21:49:45 +00:00
|
|
|
|
2014-10-09 10:28:37 +00:00
|
|
|
addLiteralChar(c);
|
|
|
|
return true;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2018-09-06 09:05:35 +00:00
|
|
|
bool Scanner::isUnicodeLinebreak()
|
|
|
|
{
|
|
|
|
if (0x0a <= m_char && m_char <= 0x0d)
|
|
|
|
// line feed, vertical tab, form feed, carriage return
|
|
|
|
return true;
|
|
|
|
else if (!m_source.isPastEndOfInput(1) && uint8_t(m_source.get(0)) == 0xc2 && uint8_t(m_source.get(1)) == 0x85)
|
|
|
|
// NEL - U+0085, C2 85 in utf8
|
|
|
|
return true;
|
|
|
|
else if (!m_source.isPastEndOfInput(2) && uint8_t(m_source.get(0)) == 0xe2 && uint8_t(m_source.get(1)) == 0x80 && (
|
|
|
|
uint8_t(m_source.get(2)) == 0xa8 || uint8_t(m_source.get(2)) == 0xa9
|
|
|
|
))
|
|
|
|
// LS - U+2028, E2 80 A8 in utf8
|
|
|
|
// PS - U+2029, E2 80 A9 in utf8
|
|
|
|
return true;
|
|
|
|
else
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2014-10-06 15:13:52 +00:00
|
|
|
Token::Value Scanner::scanString()
|
|
|
|
{
|
2014-10-20 11:02:06 +00:00
|
|
|
char const quote = m_char;
|
2014-10-09 10:28:37 +00:00
|
|
|
advance(); // consume quote
|
2014-11-27 17:57:50 +00:00
|
|
|
LiteralScope literal(this, LITERAL_TYPE_STRING);
|
2018-09-06 09:05:35 +00:00
|
|
|
while (m_char != quote && !isSourcePastEndOfInput() && !isUnicodeLinebreak())
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
char c = m_char;
|
|
|
|
advance();
|
2014-10-16 12:08:54 +00:00
|
|
|
if (c == '\\')
|
|
|
|
{
|
|
|
|
if (isSourcePastEndOfInput() || !scanEscape())
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal;
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
|
|
|
addLiteralChar(c);
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-11-21 16:08:35 +00:00
|
|
|
if (m_char != quote)
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal;
|
2014-11-21 16:08:35 +00:00
|
|
|
literal.complete();
|
2014-10-09 10:28:37 +00:00
|
|
|
advance(); // consume quote
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::StringLiteral;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2016-08-16 14:31:23 +00:00
|
|
|
Token::Value Scanner::scanHexString()
|
|
|
|
{
|
|
|
|
char const quote = m_char;
|
|
|
|
advance(); // consume quote
|
|
|
|
LiteralScope literal(this, LITERAL_TYPE_STRING);
|
2018-09-06 09:05:35 +00:00
|
|
|
while (m_char != quote && !isSourcePastEndOfInput())
|
2016-08-16 14:31:23 +00:00
|
|
|
{
|
|
|
|
char c = m_char;
|
|
|
|
if (!scanHexByte(c))
|
|
|
|
return Token::Illegal;
|
|
|
|
addLiteralChar(c);
|
|
|
|
}
|
|
|
|
if (m_char != quote)
|
|
|
|
return Token::Illegal;
|
|
|
|
literal.complete();
|
|
|
|
advance(); // consume quote
|
|
|
|
return Token::StringLiteral;
|
|
|
|
}
|
|
|
|
|
2018-08-03 14:13:52 +00:00
|
|
|
// Parse for regex [:digit:]+(_[:digit:]+)*
|
2014-10-06 15:13:52 +00:00
|
|
|
void Scanner::scanDecimalDigits()
|
|
|
|
{
|
2018-08-03 14:13:52 +00:00
|
|
|
// MUST begin with a decimal digit.
|
|
|
|
if (!isDecimalDigit(m_char))
|
|
|
|
return;
|
2017-10-25 08:12:07 +00:00
|
|
|
|
2018-08-03 14:13:52 +00:00
|
|
|
// May continue with decimal digit or underscore for grouping.
|
|
|
|
do addLiteralCharAndAdvance();
|
|
|
|
while (!m_source.isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
|
2017-10-25 08:12:07 +00:00
|
|
|
|
2018-08-03 14:13:52 +00:00
|
|
|
// Defer further validation of underscore to SyntaxChecker.
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2014-11-05 07:40:21 +00:00
|
|
|
Token::Value Scanner::scanNumber(char _charSeen)
|
2014-10-06 15:13:52 +00:00
|
|
|
{
|
2014-11-05 07:40:21 +00:00
|
|
|
enum { DECIMAL, HEX, BINARY } kind = DECIMAL;
|
2014-11-27 17:57:50 +00:00
|
|
|
LiteralScope literal(this, LITERAL_TYPE_NUMBER);
|
2014-11-05 07:40:21 +00:00
|
|
|
if (_charSeen == '.')
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
|
|
|
// we have already seen a decimal point of the float
|
|
|
|
addLiteralChar('.');
|
2018-08-03 14:13:52 +00:00
|
|
|
if (m_char == '_')
|
|
|
|
return Token::Illegal;
|
2014-10-16 12:08:54 +00:00
|
|
|
scanDecimalDigits(); // we know we have at least one digit
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2014-12-19 10:31:17 +00:00
|
|
|
solAssert(_charSeen == 0, "");
|
2014-10-16 12:08:54 +00:00
|
|
|
// if the first character is '0' we must check for octals and hex
|
|
|
|
if (m_char == '0')
|
|
|
|
{
|
|
|
|
addLiteralCharAndAdvance();
|
2014-11-05 07:40:21 +00:00
|
|
|
// either 0, 0exxx, 0Exxx, 0.xxx or a hex number
|
2018-09-18 17:09:16 +00:00
|
|
|
if (m_char == 'x')
|
2014-10-16 12:08:54 +00:00
|
|
|
{
|
|
|
|
// hex number
|
|
|
|
kind = HEX;
|
|
|
|
addLiteralCharAndAdvance();
|
2014-11-21 16:08:35 +00:00
|
|
|
if (!isHexDigit(m_char))
|
2018-09-18 17:09:16 +00:00
|
|
|
return Token::Illegal; // we must have at least one hex digit after 'x'
|
2017-10-25 08:12:07 +00:00
|
|
|
|
2018-08-03 14:13:52 +00:00
|
|
|
while (isHexDigit(m_char) || m_char == '_') // We keep the underscores for later validation
|
2014-10-16 12:08:54 +00:00
|
|
|
addLiteralCharAndAdvance();
|
|
|
|
}
|
2017-03-07 11:34:32 +00:00
|
|
|
else if (isDecimalDigit(m_char))
|
|
|
|
// We do not allow octal numbers
|
|
|
|
return Token::Illegal;
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
// Parse decimal digits and allow trailing fractional part.
|
|
|
|
if (kind == DECIMAL)
|
|
|
|
{
|
|
|
|
scanDecimalDigits(); // optional
|
|
|
|
if (m_char == '.')
|
|
|
|
{
|
2018-08-03 14:13:52 +00:00
|
|
|
if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
|
|
|
|
{
|
|
|
|
// Assume the input may be a floating point number with leading '_' in fraction part.
|
|
|
|
// Recover by consuming it all but returning `Illegal` right away.
|
|
|
|
addLiteralCharAndAdvance(); // '.'
|
|
|
|
addLiteralCharAndAdvance(); // '_'
|
|
|
|
scanDecimalDigits();
|
|
|
|
}
|
2018-05-22 15:37:21 +00:00
|
|
|
if (m_source.isPastEndOfInput() || !isDecimalDigit(m_source.get(1)))
|
|
|
|
{
|
2018-08-03 14:13:52 +00:00
|
|
|
// A '.' has to be followed by a number.
|
2018-05-22 15:37:21 +00:00
|
|
|
literal.complete();
|
|
|
|
return Token::Number;
|
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
addLiteralCharAndAdvance();
|
2018-05-22 15:37:21 +00:00
|
|
|
scanDecimalDigits();
|
2014-10-16 12:08:54 +00:00
|
|
|
}
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
// scan exponent, if any
|
|
|
|
if (m_char == 'e' || m_char == 'E')
|
|
|
|
{
|
2014-12-17 15:23:18 +00:00
|
|
|
solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
|
2014-11-21 16:08:35 +00:00
|
|
|
if (kind != DECIMAL)
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal;
|
2018-08-03 14:13:52 +00:00
|
|
|
else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
|
|
|
|
{
|
|
|
|
// Recover from wrongly placed underscore as delimiter in literal with scientific
|
|
|
|
// notation by consuming until the end.
|
|
|
|
addLiteralCharAndAdvance(); // 'e'
|
|
|
|
addLiteralCharAndAdvance(); // '_'
|
|
|
|
scanDecimalDigits();
|
|
|
|
literal.complete();
|
|
|
|
return Token::Number;
|
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
// scan exponent
|
2018-08-03 14:13:52 +00:00
|
|
|
addLiteralCharAndAdvance(); // 'e' | 'E'
|
2014-10-16 12:08:54 +00:00
|
|
|
if (m_char == '+' || m_char == '-')
|
|
|
|
addLiteralCharAndAdvance();
|
2014-11-21 16:08:35 +00:00
|
|
|
if (!isDecimalDigit(m_char))
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal; // we must have at least one decimal digit after 'e'/'E'
|
2014-10-16 12:08:54 +00:00
|
|
|
scanDecimalDigits();
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-10-16 12:08:54 +00:00
|
|
|
// The source character immediately following a numeric literal must
|
|
|
|
// not be an identifier start or a decimal digit; see ECMA-262
|
|
|
|
// section 7.8.3, page 17 (note that we read only one decimal digit
|
|
|
|
// if the value is 0).
|
2016-02-09 21:43:23 +00:00
|
|
|
if (isDecimalDigit(m_char) || isIdentifierStart(m_char))
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Illegal;
|
2014-11-21 16:08:35 +00:00
|
|
|
literal.complete();
|
2015-02-09 13:00:12 +00:00
|
|
|
return Token::Number;
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2016-02-12 21:01:27 +00:00
|
|
|
tuple<Token::Value, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
|
2014-10-06 15:13:52 +00:00
|
|
|
{
|
2016-02-09 21:43:23 +00:00
|
|
|
solAssert(isIdentifierStart(m_char), "");
|
2014-11-27 17:57:50 +00:00
|
|
|
LiteralScope literal(this, LITERAL_TYPE_STRING);
|
2014-10-09 10:28:37 +00:00
|
|
|
addLiteralCharAndAdvance();
|
|
|
|
// Scan the rest of the identifier characters.
|
2016-02-09 21:43:23 +00:00
|
|
|
while (isIdentifierPart(m_char)) //get full literal
|
2014-10-09 10:28:37 +00:00
|
|
|
addLiteralCharAndAdvance();
|
2014-11-21 16:08:35 +00:00
|
|
|
literal.complete();
|
2016-02-09 21:43:23 +00:00
|
|
|
return Token::fromIdentifierOrKeyword(m_nextToken.literal);
|
2014-10-06 15:13:52 +00:00
|
|
|
}
|
|
|
|
|
2014-11-30 21:43:40 +00:00
|
|
|
char CharStream::advanceAndGet(size_t _chars)
|
2014-10-16 21:49:45 +00:00
|
|
|
{
|
2014-10-20 12:00:37 +00:00
|
|
|
if (isPastEndOfInput())
|
|
|
|
return 0;
|
2015-08-31 16:44:29 +00:00
|
|
|
m_position += _chars;
|
2014-10-20 12:00:37 +00:00
|
|
|
if (isPastEndOfInput())
|
|
|
|
return 0;
|
2015-08-31 16:44:29 +00:00
|
|
|
return m_source[m_position];
|
2014-11-27 17:57:50 +00:00
|
|
|
}
|
|
|
|
|
2014-10-16 21:49:45 +00:00
|
|
|
char CharStream::rollback(size_t _amount)
|
|
|
|
{
|
2015-08-31 16:44:29 +00:00
|
|
|
solAssert(m_position >= _amount, "");
|
|
|
|
m_position -= _amount;
|
2014-10-16 21:49:45 +00:00
|
|
|
return get();
|
|
|
|
}
|
|
|
|
|
2015-08-31 16:44:29 +00:00
|
|
|
string CharStream::lineAtPosition(int _position) const
|
2014-10-08 18:53:50 +00:00
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
// if _position points to \n, it returns the line before the \n
|
2014-10-24 17:06:30 +00:00
|
|
|
using size_type = string::size_type;
|
|
|
|
size_type searchStart = min<size_type>(m_source.size(), _position);
|
2014-10-20 12:00:37 +00:00
|
|
|
if (searchStart > 0)
|
|
|
|
searchStart--;
|
2014-10-09 10:28:37 +00:00
|
|
|
size_type lineStart = m_source.rfind('\n', searchStart);
|
2014-10-24 17:06:30 +00:00
|
|
|
if (lineStart == string::npos)
|
2014-10-09 10:28:37 +00:00
|
|
|
lineStart = 0;
|
|
|
|
else
|
|
|
|
lineStart++;
|
2014-10-24 17:06:30 +00:00
|
|
|
return m_source.substr(lineStart, min(m_source.find('\n', lineStart),
|
|
|
|
m_source.size()) - lineStart);
|
2014-10-08 18:53:50 +00:00
|
|
|
}
|
|
|
|
|
2014-10-24 17:06:30 +00:00
|
|
|
tuple<int, int> CharStream::translatePositionToLineColumn(int _position) const
|
2014-10-08 18:53:50 +00:00
|
|
|
{
|
2014-10-24 17:06:30 +00:00
|
|
|
using size_type = string::size_type;
|
|
|
|
size_type searchPosition = min<size_type>(m_source.size(), _position);
|
|
|
|
int lineNumber = count(m_source.begin(), m_source.begin() + searchPosition, '\n');
|
2014-10-09 10:28:37 +00:00
|
|
|
size_type lineStart;
|
2014-10-16 12:08:54 +00:00
|
|
|
if (searchPosition == 0)
|
2014-10-09 10:28:37 +00:00
|
|
|
lineStart = 0;
|
2014-10-16 12:08:54 +00:00
|
|
|
else
|
|
|
|
{
|
2014-10-09 10:28:37 +00:00
|
|
|
lineStart = m_source.rfind('\n', searchPosition - 1);
|
2014-10-24 17:06:30 +00:00
|
|
|
lineStart = lineStart == string::npos ? 0 : lineStart + 1;
|
2014-10-09 10:28:37 +00:00
|
|
|
}
|
2014-10-24 17:06:30 +00:00
|
|
|
return tuple<int, int>(lineNumber, searchPosition - lineStart);
|
2014-10-08 18:53:50 +00:00
|
|
|
}
|
|
|
|
|
2014-10-06 15:13:52 +00:00
|
|
|
|
2014-10-16 12:08:54 +00:00
|
|
|
}
|
|
|
|
}
|