cerc-io/solidity
16
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #9526 from ethereum/scanner-split

Browse Source 
Add ScannerKind and replace supportPeriodInIdentifier
This commit is contained in:
chriseth 2020-07-28 23:25:28 +02:00 committed by GitHub
commit 9e5a56a649
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 23 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
liblangutil/Scanner.cpp
View File

@ -147,7 +147,7 @@ void Scanner::reset(shared_ptr<CharStream> _source)
void Scanner::reset()
{
m_source->reset();
m_supportPeriodInIdentifier = false;
m_kind = ScannerKind::Solidity;
m_char = m_source->get();
skipWhitespace();
next();
@ -163,12 +163,6 @@ void Scanner::setPosition(size_t _offset)
next();
}
void Scanner::supportPeriodInIdentifier(bool _value)
{
m_supportPeriodInIdentifier = _value;
rescan();
}
bool Scanner::scanHexByte(char& o_scannedByte)
{
char x = 0;
@ -970,7 +964,7 @@ tuple<Token, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
LiteralScope literal(this, LITERAL_TYPE_STRING);
addLiteralCharAndAdvance();
// Scan the rest of the identifier characters.
while (isIdentifierPart(m_char) || (m_char == '.' && m_supportPeriodInIdentifier))
while (isIdentifierPart(m_char) || (m_char == '.' && m_kind == ScannerKind::Yul))
addLiteralCharAndAdvance();
literal.complete();
return TokenTraits::fromIdentifierOrKeyword(m_tokens[NextNext].literal);

21
liblangutil/Scanner.h
View File

@ -68,6 +68,12 @@ class AstRawString;
class AstValueFactory;
class ParserRecorder;
enum class ScannerKind
{
Solidity,
Yul
};
enum class ScannerError
{
NoError,
@ -107,9 +113,14 @@ public:
/// Resets scanner to the start of input.
void reset();
/// Enables or disables support for period in identifier.
/// This re-scans the current token and comment literal and thus invalidates it.
void supportPeriodInIdentifier(bool _value);
/// Changes the scanner mode.
void setScannerMode(ScannerKind _kind)
{
m_kind = _kind;
// Invalidate lookahead buffer.
rescan();
}
/// @returns the next token and advances input
Token next();
@ -249,8 +260,6 @@ private:
size_t sourcePos() const { return m_source->position(); }
bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
bool m_supportPeriodInIdentifier = false;
enum TokenIndex { Current, Next, NextNext };
TokenDesc m_skippedComments[3] = {}; // desc for the current, next and nextnext skipped comment
@ -258,6 +267,8 @@ private:
std::shared_ptr<CharStream> m_source;
ScannerKind m_kind = ScannerKind::Solidity;
/// one character look-ahead, equals 0 at end of input
char m_char;
};

4
libyul/AsmParser.cpp
View File

@ -42,8 +42,8 @@ unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _
{
m_recursionDepth = 0;
_scanner->supportPeriodInIdentifier(true);
ScopeGuard resetScanner([&]{ _scanner->supportPeriodInIdentifier(false); });
_scanner->setScannerMode(ScannerKind::Yul);
ScopeGuard resetScanner([&]{ _scanner->setScannerMode(ScannerKind::Solidity); });
try
{

6
test/libsolidity/SolidityScanner.cpp
View File

@ -214,7 +214,7 @@ BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function .a(", ""));
scanner.supportPeriodInIdentifier(true);
scanner.setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -233,7 +233,7 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function a...a(", ""));
scanner.supportPeriodInIdentifier(true);
scanner.setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@ -249,7 +249,7 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function a.(", ""));
scanner.supportPeriodInIdentifier(true);
scanner.setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);