cerc-io/solidity
11
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove scanner from compiler stack.

Browse Source
This commit is contained in:
chriseth
2021-08-03 15:43:17 +02:00
committed by Christian Parpart
parent af18b8afc2
commit ffc5cfd9a5
23 changed files with 275 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/liblangutil/Scanner.cpp
+160 -124
View File
@@ -34,13 +34,15 @@ BOOST_AUTO_TEST_SUITE(ScannerTest)
BOOST_AUTO_TEST_CASE(test_empty)
{
Scanner scanner(CharStream{});
CharStream stream{};
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(smoke_test)
{
Scanner scanner(CharStream("function break;765 \t \"string1\",'string2'\nidentifier1", ""));
CharStream stream("function break;765 \t \"string1\",'string2'\nidentifier1", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Break);
BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
@@ -58,7 +60,8 @@ BOOST_AUTO_TEST_CASE(smoke_test)
BOOST_AUTO_TEST_CASE(assembly_assign)
{
Scanner scanner(CharStream("let a := 1", ""));
CharStream stream("let a := 1", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Let);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
@@ -69,7 +72,8 @@ BOOST_AUTO_TEST_CASE(assembly_assign)
BOOST_AUTO_TEST_CASE(assembly_multiple_assign)
{
Scanner scanner(CharStream("let a, b, c := 1", ""));
CharStream stream("let a, b, c := 1", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Let);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
@@ -89,14 +93,16 @@ BOOST_AUTO_TEST_CASE(string_printable)
// Escape \ and " (since we are quoting with ")
if (v == '\\' || v == '"')
lit = string{'\\'} + lit;
Scanner scanner(CharStream(" { \"" + lit + "\"", ""));
CharStream stream(" { \"" + lit + "\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), string{static_cast<char>(v)});
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
// Special case of unescaped " for strings quoted with '
Scanner scanner(CharStream(" { '\"'", ""));
CharStream stream(" { '\"'", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "\"");
@@ -110,7 +116,8 @@ BOOST_AUTO_TEST_CASE(string_nonprintable)
if (v >= 0x20 && v <= 0x7e)
continue;
string lit{static_cast<char>(v)};
Scanner scanner(CharStream(" { \"" + lit + "\"", ""));
CharStream stream(" { \"" + lit + "\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
if (v == '\n' || v == '\v' || v == '\f' || v == '\r')
@@ -123,7 +130,8 @@ BOOST_AUTO_TEST_CASE(string_nonprintable)
BOOST_AUTO_TEST_CASE(string_escapes)
{
Scanner scanner(CharStream(" { \"a\\x61\"", ""));
CharStream stream(" { \"a\\x61\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa");
@@ -131,25 +139,46 @@ BOOST_AUTO_TEST_CASE(string_escapes)
BOOST_AUTO_TEST_CASE(string_escapes_all)
{
Scanner scanner(CharStream(" { \"a\\x61\\n\\r\\t\"", ""));
CharStream stream(" { \"a\\x61\\n\\r\\t\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa\n\r\t");
}
struct TestScanner
{
unique_ptr<CharStream> stream;
unique_ptr<Scanner> scanner;
explicit TestScanner(string _text) { reset(move(_text)); }
void reset(std::string _text)
{
stream = make_unique<CharStream>(move(_text), "");
scanner = make_unique<Scanner>(*stream);
}
decltype(auto) currentToken() { return scanner->currentToken(); }
decltype(auto) next() { return scanner->next(); }
decltype(auto) currentError() { return scanner->currentError(); }
decltype(auto) currentLiteral() { return scanner->currentLiteral(); }
decltype(auto) currentCommentLiteral() { return scanner->currentCommentLiteral(); }
decltype(auto) currentLocation() { return scanner->currentLocation(); }
};
BOOST_AUTO_TEST_CASE(string_escapes_legal_before_080)
{
Scanner scanner(CharStream(" { \"a\\b", ""));
TestScanner scanner(" { \"a\\b");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
scanner.reset(CharStream(" { \"a\\f", ""));
scanner.reset(" { \"a\\f");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
scanner.reset(CharStream(" { \"a\\v", ""));
scanner.reset(" { \"a\\v");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@@ -158,7 +187,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_legal_before_080)
BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
{
Scanner scanner(CharStream(" { \"a\\x61\\x00abc\"", ""));
TestScanner scanner(" { \"a\\x61\\x00abc\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("aa\0abc", 6));
@@ -166,7 +195,8 @@ BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
BOOST_AUTO_TEST_CASE(string_escape_illegal)
{
Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)", ""));
CharStream stream(" bla \"\\x6rf\" (illegalescape)", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@@ -180,7 +210,7 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal)
BOOST_AUTO_TEST_CASE(hex_numbers)
{
Scanner scanner(CharStream("var x = 0x765432536763762734623472346;", ""));
TestScanner scanner("var x = 0x765432536763762734623472346;");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -188,34 +218,35 @@ BOOST_AUTO_TEST_CASE(hex_numbers)
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x765432536763762734623472346");
BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("0x1234", ""));
scanner.reset("0x1234");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234");
scanner.reset(CharStream("0X1234", ""));
scanner.reset("0X1234");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
}
BOOST_AUTO_TEST_CASE(octal_numbers)
{
Scanner scanner(CharStream("07", ""));
TestScanner scanner("07");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
scanner.reset(CharStream("007", ""));
scanner.reset("007");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
scanner.reset(CharStream("-07", ""));
scanner.reset("-07");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
scanner.reset(CharStream("-.07", ""));
scanner.reset("-.07");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
scanner.reset(CharStream("0", ""));
scanner.reset("0");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
scanner.reset(CharStream("0.1", ""));
scanner.reset("0.1");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
}
BOOST_AUTO_TEST_CASE(scientific_notation)
{
Scanner scanner(CharStream("var x = 2e10;", ""));
CharStream stream("var x = 2e10;", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -227,14 +258,14 @@ BOOST_AUTO_TEST_CASE(scientific_notation)
BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
{
Scanner scanner(CharStream("function .a(", ""));
TestScanner scanner("function .a(");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function .a(", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("function .a(");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -244,7 +275,7 @@ BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
{
Scanner scanner(CharStream("function a..a(", ""));
TestScanner scanner("function a..a(");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
@@ -252,8 +283,8 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function a...a(", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("function a...a(");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -262,14 +293,14 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
{
Scanner scanner(CharStream("function a.(", ""));
TestScanner scanner("function a.(");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function a.(", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("function a.(");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -278,19 +309,19 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
BOOST_AUTO_TEST_CASE(trailing_dot_in_numbers)
{
Scanner scanner(CharStream("2.5", ""));
TestScanner scanner("2.5");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("2.5e10", ""));
scanner.reset("2.5e10");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(".5", ""));
scanner.reset(".5");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(".5e10", ""));
scanner.reset(".5e10");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("2.", ""));
scanner.reset("2.");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -299,7 +330,8 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_numbers)
BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("_1.2", ""));
CharStream stream("_1.2", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -308,11 +340,11 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("1._2", ""));
TestScanner scanner("1._2");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("1._", ""));
scanner.reset("1._");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@@ -320,7 +352,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("_1e2", ""));
CharStream stream("_1e2", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@@ -328,7 +361,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("1e_2", ""));
CharStream stream("1e_2", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2");
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -336,7 +370,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
{
Scanner scanner(CharStream("0x_abc", ""));
CharStream stream("0x_abc", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -345,7 +380,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("12._1234_1234", ""));
CharStream stream("12._1234_1234", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@@ -353,22 +389,22 @@ BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("0x123_", ""));
TestScanner scanner("0x123_");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("123_", ""));
scanner.reset("123_");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("12.34_", ""));
scanner.reset("12.34_");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(negative_numbers)
{
Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;", ""));
TestScanner scanner("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -395,7 +431,7 @@ BOOST_AUTO_TEST_CASE(negative_numbers)
BOOST_AUTO_TEST_CASE(locations)
{
Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment", ""));
TestScanner scanner("function_identifier has ; -0x743/*comment*/\n ident //comment");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLocation().start, 0);
BOOST_CHECK_EQUAL(scanner.currentLocation().end, 19);
@@ -418,7 +454,7 @@ BOOST_AUTO_TEST_CASE(locations)
BOOST_AUTO_TEST_CASE(ambiguities)
{
// test scanning of some operators which need look-ahead
Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=", ""));
TestScanner scanner("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LessThanOrEqual);
BOOST_CHECK_EQUAL(scanner.next(), Token::LessThan);
BOOST_CHECK_EQUAL(scanner.next(), Token::Add);
@@ -440,21 +476,21 @@ BOOST_AUTO_TEST_CASE(ambiguities)
BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin)
{
Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user", ""));
TestScanner scanner("/// Send $(value / 1000) chocolates to the user");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
}
BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed_begin)
{
Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/", ""));
TestScanner scanner("/** Send $(value / 1000) chocolates to the user*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
}
BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
{
Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user", ""));
TestScanner scanner("some other tokens /// Send $(value / 1000) chocolates to the user");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -464,9 +500,9 @@ BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
{
Scanner scanner(CharStream("some other tokens /**\n"
"* Send $(value / 1000) chocolates to the user\n"
"*/", ""));
TestScanner scanner("some other tokens /**\n"
"* Send $(value / 1000) chocolates to the user\n"
"*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -476,9 +512,9 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
{
Scanner scanner(CharStream("some other tokens /**\n"
" Send $(value / 1000) chocolates to the user\n"
"*/", ""));
TestScanner scanner("some other tokens /**\n"
" Send $(value / 1000) chocolates to the user\n"
"*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -488,9 +524,9 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
{
Scanner scanner(CharStream("some other tokens /** \t \r \n"
"\t \r * Send $(value / 1000) chocolates to the user\n"
"*/", ""));
TestScanner scanner("some other tokens /** \t \r \n"
"\t \r * Send $(value / 1000) chocolates to the user\n"
"*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -500,37 +536,37 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
BOOST_AUTO_TEST_CASE(comment_before_eos)
{
Scanner scanner(CharStream("//", ""));
TestScanner scanner("//");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(documentation_comment_before_eos)
{
Scanner scanner(CharStream("///", ""));
TestScanner scanner("///");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(empty_multiline_comment)
{
Scanner scanner(CharStream("/**/", ""));
TestScanner scanner("/**/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(empty_multiline_documentation_comment_before_eos)
{
Scanner scanner(CharStream("/***/", ""));
TestScanner scanner("/***/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
{
Scanner scanner(CharStream("hello_world ///documentation comment \n"
"//simple comment \n"
"<<", ""));
TestScanner scanner("hello_world ///documentation comment \n"
"//simple comment \n"
"<<");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::SHL);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "documentation comment ");
@@ -538,7 +574,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
BOOST_AUTO_TEST_CASE(ether_subdenominations)
{
Scanner scanner(CharStream("wei gwei ether", ""));
TestScanner scanner("wei gwei ether");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubWei);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubGwei);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubEther);
@@ -546,7 +582,7 @@ BOOST_AUTO_TEST_CASE(ether_subdenominations)
BOOST_AUTO_TEST_CASE(time_subdenominations)
{
Scanner scanner(CharStream("seconds minutes hours days weeks years", ""));
TestScanner scanner("seconds minutes hours days weeks years");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubSecond);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubMinute);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubHour);
@@ -557,7 +593,7 @@ BOOST_AUTO_TEST_CASE(time_subdenominations)
BOOST_AUTO_TEST_CASE(empty_comment)
{
Scanner scanner(CharStream("//\ncontract{}", ""));
TestScanner scanner("//\ncontract{}");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Contract);
BOOST_CHECK_EQUAL(scanner.next(), Token::LBrace);
@@ -569,7 +605,7 @@ BOOST_AUTO_TEST_CASE(empty_comment)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
{
Scanner scanner(CharStream("{ \"\\u00DAnicode\"", ""));
TestScanner scanner("{ \"\\u00DAnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xC3\x9Anicode", 8));
@@ -577,7 +613,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
{
Scanner scanner(CharStream("{ \"\\u007Fnicode\"", ""));
TestScanner scanner("{ \"\\u007Fnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x7Fnicode", 7));
@@ -585,7 +621,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
{
Scanner scanner(CharStream("{ \"\\u07FFnicode\"", ""));
TestScanner scanner("{ \"\\u07FFnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xDF\xBFnicode", 8));
@@ -593,7 +629,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
{
Scanner scanner(CharStream("{ \"\\uFFFFnicode\"", ""));
TestScanner scanner("{ \"\\uFFFFnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xEF\xBF\xBFnicode", 9));
@@ -601,7 +637,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
{
Scanner scanner(CharStream("{ \"\\uFFnicode\"", ""));
TestScanner scanner("{ \"\\uFFnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
}
@@ -610,12 +646,12 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
BOOST_AUTO_TEST_CASE(unicode_prefix_only)
{
Scanner scanner(CharStream("{ unicode", ""));
TestScanner scanner("{ unicode");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ unicode", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ unicode");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "unicode");
@@ -623,7 +659,7 @@ BOOST_AUTO_TEST_CASE(unicode_prefix_only)
BOOST_AUTO_TEST_CASE(unicode_invalid_space)
{
Scanner scanner(CharStream("{ unicode ", ""));
TestScanner scanner("{ unicode ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@@ -631,12 +667,12 @@ BOOST_AUTO_TEST_CASE(unicode_invalid_space)
BOOST_AUTO_TEST_CASE(unicode_invalid_token)
{
Scanner scanner(CharStream("{ unicode test", ""));
TestScanner scanner("{ unicode test");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ unicode test", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ unicode test");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "unicode");
@@ -646,7 +682,7 @@ BOOST_AUTO_TEST_CASE(unicode_invalid_token)
BOOST_AUTO_TEST_CASE(valid_unicode_literal)
{
Scanner scanner(CharStream("{ unicode\"Hello 😃\"", ""));
TestScanner scanner("{ unicode\"Hello 😃\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::UnicodeStringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("Hello \xf0\x9f\x98\x83", 10));
@@ -655,7 +691,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_literal)
BOOST_AUTO_TEST_CASE(valid_nonprintable_in_unicode_literal)
{
// Non-printable characters are allowed in unicode strings...
Scanner scanner(CharStream("{ unicode\"Hello \007😃\"", ""));
TestScanner scanner("{ unicode\"Hello \007😃\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::UnicodeStringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("Hello \x07\xf0\x9f\x98\x83", 11));
@@ -665,19 +701,19 @@ BOOST_AUTO_TEST_CASE(valid_nonprintable_in_unicode_literal)
BOOST_AUTO_TEST_CASE(hex_prefix_only)
{
Scanner scanner(CharStream("{ hex", ""));
TestScanner scanner("{ hex");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ hex", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ hex");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
}
BOOST_AUTO_TEST_CASE(hex_invalid_space)
{
Scanner scanner(CharStream("{ hex ", ""));
TestScanner scanner("{ hex ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@@ -685,12 +721,12 @@ BOOST_AUTO_TEST_CASE(hex_invalid_space)
BOOST_AUTO_TEST_CASE(hex_invalid_token)
{
Scanner scanner(CharStream("{ hex test", ""));
TestScanner scanner("{ hex test");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ hex test", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ hex test");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@@ -698,7 +734,7 @@ BOOST_AUTO_TEST_CASE(hex_invalid_token)
BOOST_AUTO_TEST_CASE(valid_hex_literal)
{
Scanner scanner(CharStream("{ hex\"00112233FF\"", ""));
TestScanner scanner("{ hex\"00112233FF\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::HexStringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x00\x11\x22\x33\xFF", 5));
@@ -706,7 +742,7 @@ BOOST_AUTO_TEST_CASE(valid_hex_literal)
BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
{
Scanner scanner(CharStream("{ hex\"00112233F\"", ""));
TestScanner scanner("{ hex\"00112233F\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -714,7 +750,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
{
Scanner scanner(CharStream("{ hex\"00112233FF \"", ""));
TestScanner scanner("{ hex\"00112233FF \"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -722,7 +758,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
{
Scanner scanner(CharStream("{ hex\"00112233FF'", ""));
TestScanner scanner("{ hex\"00112233FF'");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -730,7 +766,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
{
Scanner scanner(CharStream("{ hex\"hello\"", ""));
TestScanner scanner("{ hex\"hello\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -741,7 +777,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
{
// This used to parse as "comment", "identifier"
Scanner scanner(CharStream("/** / x", ""));
TestScanner scanner("/** / x");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@@ -749,14 +785,14 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos)
{
// This used to parse as "whitespace"
Scanner scanner(CharStream("/**", ""));
TestScanner scanner("/**");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(multiline_comment_at_eos)
{
Scanner scanner(CharStream("/*", ""));
TestScanner scanner("/*");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@@ -765,7 +801,7 @@ BOOST_AUTO_TEST_CASE(regular_line_break_in_single_line_comment)
{
for (auto const& nl: {"\r", "\n", "\r\n"})
{
Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
TestScanner scanner("// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -777,7 +813,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_comment)
{
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{
Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
TestScanner scanner("// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size() - 1; i++)
@@ -792,7 +828,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_single_line_doc_comment)
{
for (auto const& nl: {"\r", "\n", "\r\n"})
{
Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
TestScanner scanner("/// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -806,7 +842,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_multiline_doc_comment)
// Any accepted non-LF is being canonicalized to LF.
for (auto const& nl : {"\r"s, "\n"s, "\r\n"s})
{
Scanner scanner{CharStream{"/// Hello" + nl + "/// World" + nl + "ident", ""}};
TestScanner scanner{"/// Hello" + nl + "/// World" + nl + "ident"};
auto const& lit = scanner.currentCommentLiteral();
BOOST_CHECK_EQUAL(lit, "Hello\n World");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Hello\n World");
@@ -820,7 +856,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_doc_comment)
{
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{
Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
TestScanner scanner("/// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size() - 1; i++)
@@ -835,7 +871,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings)
{
for (auto const& nl: {"\r"s, "\n"s, "\r\n"s})
{
Scanner scanner(CharStream("\"abc " + nl + " def\"", ""));
TestScanner scanner("\"abc " + nl + " def\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -848,7 +884,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings)
{
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{
Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
TestScanner scanner("\"abc " + string(nl) + " def\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size(); i++)
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
@@ -863,7 +899,7 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
{
// These are tokens which have a different meaning in Yul.
string keywords = "return byte bool address var in true false leave switch case default";
Scanner scanner(CharStream(keywords, ""));
TestScanner scanner(keywords);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Return);
BOOST_CHECK_EQUAL(scanner.next(), Token::Byte);
BOOST_CHECK_EQUAL(scanner.next(), Token::Bool);
@@ -877,8 +913,8 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
BOOST_CHECK_EQUAL(scanner.next(), Token::Case);
BOOST_CHECK_EQUAL(scanner.next(), Token::Default);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(keywords, ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset(keywords);
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -896,28 +932,28 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
BOOST_AUTO_TEST_CASE(yul_keyword_like)
{
Scanner scanner(CharStream("leave.function", ""));
TestScanner scanner("leave.function");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("leave.function", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("leave.function");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(yul_identifier_with_dots)
{
Scanner scanner(CharStream("mystorage.slot := 1", ""));
TestScanner scanner("mystorage.slot := 1");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("mystorage.slot := 1", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("mystorage.slot := 1");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
@@ -927,7 +963,7 @@ BOOST_AUTO_TEST_CASE(yul_identifier_with_dots)
BOOST_AUTO_TEST_CASE(yul_function)
{
string sig = "function f(a, b) -> x, y";
Scanner scanner(CharStream(sig, ""));
TestScanner scanner(sig);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -940,8 +976,8 @@ BOOST_AUTO_TEST_CASE(yul_function)
BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(sig, ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset(sig);
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -959,7 +995,7 @@ BOOST_AUTO_TEST_CASE(yul_function)
BOOST_AUTO_TEST_CASE(yul_function_with_whitespace)
{
string sig = "function f (a, b) - > x, y";
Scanner scanner(CharStream(sig, ""));
TestScanner scanner(sig);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -973,8 +1009,8 @@ BOOST_AUTO_TEST_CASE(yul_function_with_whitespace)
BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(sig, ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset(sig);
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
test/libsolidity/Assembly.cpp
+3 -2
View File
@@ -26,7 +26,8 @@
#include <liblangutil/SourceLocation.h>
#include <libevmasm/Assembly.h>
#include <liblangutil/Scanner.h>
#include <liblangutil/CharStream.h>
#include <libsolidity/parsing/Parser.h>
#include <libsolidity/analysis/DeclarationTypeChecker.h>
#include <libsolidity/analysis/NameAndTypeResolver.h>
@@ -58,7 +59,7 @@ evmasm::AssemblyItems compileContract(std::shared_ptr<CharStream> _sourceCode)
ErrorReporter errorReporter(errors);
Parser parser(errorReporter, solidity::test::CommonOptions::get().evmVersion());
ASTPointer<SourceUnit> sourceUnit;
BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(_sourceCode)));
BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(*_sourceCode));
BOOST_CHECK(!!sourceUnit);
Scoper::assignScopes(*sourceUnit);
test/libsolidity/SemVerMatcher.cpp
+2 -1
View File
@@ -43,7 +43,8 @@ namespace
SemVerMatchExpression parseExpression(string const& _input)
{
Scanner scanner{CharStream(_input, "")};
CharStream stream(_input, "");
Scanner scanner{stream};
vector<string> literals;
vector<Token> tokens;
while (scanner.currentToken() != Token::EOS)
test/libsolidity/SolidityExpressionCompiler.cpp
+2 -3
View File
@@ -101,15 +101,14 @@ bytes compileFirstExpression(
)
{
string sourceCode = "pragma solidity >=0.0; // SPDX-License-Identifier: GPL-3\n" + _sourceCode;
CharStream stream(sourceCode, "");
ASTPointer<SourceUnit> sourceUnit;
try
{
ErrorList errors;
ErrorReporter errorReporter(errors);
sourceUnit = Parser(errorReporter, solidity::test::CommonOptions::get().evmVersion()).parse(
make_shared<Scanner>(CharStream(sourceCode, ""))
);
sourceUnit = Parser(errorReporter, solidity::test::CommonOptions::get().evmVersion()).parse(stream);
if (!sourceUnit)
return bytes();
}
test/libsolidity/SolidityParser.cpp
+2 -1
View File
@@ -42,11 +42,12 @@ namespace
ASTPointer<ContractDefinition> parseText(std::string const& _source, ErrorList& _errors, bool errorRecovery = false)
{
ErrorReporter errorReporter(_errors);
auto charStream = CharStream(_source, "");
ASTPointer<SourceUnit> sourceUnit = Parser(
errorReporter,
solidity::test::CommonOptions::get().evmVersion(),
errorRecovery
).parse(std::make_shared<Scanner>(CharStream(_source, "")));
).parse(charStream);
if (!sourceUnit)
return ASTPointer<ContractDefinition>();
for (ASTPointer<ASTNode> const& node: sourceUnit->nodes())
test/libyul/Common.cpp
+2 -1
View File
@@ -74,7 +74,8 @@ pair<shared_ptr<Object>, shared_ptr<yul::AsmAnalysisInfo>> yul::test::parse(
)
{
ErrorReporter errorReporter(_errors);
shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_source, ""));
CharStream stream(_source, "");
shared_ptr<Scanner> scanner = make_shared<Scanner>(stream);
shared_ptr<Object> parserResult = yul::ObjectParser(errorReporter, _dialect).parse(scanner, false);
if (!parserResult)
return {};
test/libyul/ObjectParser.cpp
+4 -1
View File
@@ -23,6 +23,8 @@
#include <test/libsolidity/ErrorCheck.h>
#include <liblangutil/Scanner.h>
#include <libyul/AssemblyStack.h>
#include <libyul/backends/evm/EVMDialect.h>
@@ -118,7 +120,8 @@ tuple<optional<ObjectParser::SourceNameMap>, ErrorList> tryGetSourceLocationMapp
ErrorReporter reporter(errors);
Dialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(EVMVersion::berlin());
ObjectParser objectParser{reporter, dialect};
objectParser.parse(make_shared<Scanner>(CharStream(move(source), "")), false);
CharStream stream(move(source), "");
objectParser.parse(make_shared<Scanner>(stream), false);
return {objectParser.sourceNameMapping(), std::move(errors)};
}
test/libyul/Parser.cpp
+2 -1
View File
@@ -55,7 +55,8 @@ shared_ptr<Block> parse(string const& _source, Dialect const& _dialect, ErrorRep
{
try
{
auto scanner = make_shared<Scanner>(CharStream(_source, ""));
auto stream = CharStream(_source, "");
auto scanner = make_shared<Scanner>(stream);
map<unsigned, shared_ptr<string const>> indicesToSourceNames;
indicesToSourceNames[0] = make_shared<string const>("source0");
indicesToSourceNames[1] = make_shared<string const>("source1");
test/tools/yulopti.cpp
+4 -2
View File
@@ -82,7 +82,7 @@ public:
{
SourceReferenceFormatter{
cerr,
SingletonCharStreamProvider(*m_scanner->charStream()),
SingletonCharStreamProvider(*m_charStream),
true,
false
}.printErrorInformation(m_errors);
@@ -91,7 +91,8 @@ public:
bool parse(string const& _input)
{
ErrorReporter errorReporter(m_errors);
m_scanner = make_shared<Scanner>(CharStream(_input, ""));
m_charStream = make_shared<CharStream>(_input, "");
m_scanner = make_shared<Scanner>(*m_charStream);
m_ast = yul::Parser(errorReporter, m_dialect).parse(m_scanner, false);
if (!m_ast || !errorReporter.errors().empty())
{
@@ -234,6 +235,7 @@ public:
private:
ErrorList m_errors;
shared_ptr<CharStream> m_charStream;
shared_ptr<Scanner> m_scanner;
shared_ptr<yul::Block> m_ast;
Dialect const& m_dialect{EVMDialect::strictAssemblyForEVMObjects(EVMVersion{})};