cerc-io/solidity
16
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove scanner from compiler stack.

Browse Source
This commit is contained in:
chriseth 2021-07-14 12:53:39 +02:00 committed by Christian Parpart
parent af18b8afc2
commit ffc5cfd9a5
23 changed files with 275 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
liblangutil/ParserBase.h
View File

@ -24,7 +24,6 @@
#pragma once
#include <liblangutil/Token.h>
#include <liblangutil/Scanner.h>
#include <memory>
#include <string>
@ -50,8 +49,6 @@ public:
virtual ~ParserBase() = default;
std::shared_ptr<CharStream> source() const { return m_scanner->charStream(); }
protected:
/// Utility class that creates an error and throws an exception if the
/// recursion depth is too deep.

81
liblangutil/Scanner.cpp
View File

@ -135,26 +135,11 @@ private:
bool m_complete;
};
void Scanner::reset(CharStream _source)
{
m_source = make_shared<CharStream>(std::move(_source));
m_sourceName = make_shared<string>(m_source->name());
reset();
}
void Scanner::reset(shared_ptr<CharStream> _source)
{
solAssert(_source.get() != nullptr, "You MUST provide a CharStream when resetting.");
m_source = std::move(_source);
m_sourceName = make_shared<string>(m_source->name());
reset();
}
void Scanner::reset()
{
m_source->reset();
m_source.reset();
m_kind = ScannerKind::Solidity;
m_char = m_source->get();
m_char = m_source.get();
skipWhitespace();
next();
next();
@ -163,7 +148,7 @@ void Scanner::reset()
void Scanner::setPosition(size_t _offset)
{
m_char = m_source->setPosition(_offset);
m_char = m_source.setPosition(_offset);
scanToken();
next();
next();
@ -229,7 +214,7 @@ void Scanner::rescan()
rollbackTo = static_cast<size_t>(m_tokens[Current].location.start);
else
rollbackTo = static_cast<size_t>(m_skippedComments[Current].location.start);
m_char = m_source->rollback(m_source->position() - rollbackTo);
m_char = m_source.rollback(m_source.position() - rollbackTo);
next();
next();
next();
@ -324,12 +309,12 @@ Token Scanner::skipSingleLineComment()
{
// Line terminator is not part of the comment. If it is a
// non-ascii line terminator, it will result in a parser error.
size_t startPosition = m_source->position();
size_t startPosition = m_source.position();
while (!isUnicodeLinebreak())
if (!advance())
break;
ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
if (unicodeDirectionError != ScannerError::NoError)
return setError(unicodeDirectionError);
@ -362,28 +347,28 @@ bool Scanner::tryScanEndOfLine()
size_t Scanner::scanSingleLineDocComment()
{
LiteralScope literal(this, LITERAL_TYPE_COMMENT);
size_t endPosition = m_source->position();
size_t endPosition = m_source.position();
skipWhitespaceExceptUnicodeLinebreak();
while (!isSourcePastEndOfInput())
{
endPosition = m_source->position();
endPosition = m_source.position();
if (tryScanEndOfLine())
{
// Check if next line is also a single-line comment.
// If any whitespaces were skipped, use source position before.
if (!skipWhitespaceExceptUnicodeLinebreak())
endPosition = m_source->position();
endPosition = m_source.position();
if (!m_source->isPastEndOfInput(3) &&
m_source->get(0) == '/' &&
m_source->get(1) == '/' &&
m_source->get(2) == '/')
if (!m_source.isPastEndOfInput(3) &&
m_source.get(0) == '/' &&
m_source.get(1) == '/' &&
m_source.get(2) == '/')
{
if (!m_source->isPastEndOfInput(4) && m_source->get(3) == '/')
if (!m_source.isPastEndOfInput(4) && m_source.get(3) == '/')
break; // "////" is not a documentation comment
m_char = m_source->advanceAndGet(3);
m_char = m_source.advanceAndGet(3);
if (atEndOfLine())
continue;
addCommentLiteralChar('\n');
@ -404,7 +389,7 @@ size_t Scanner::scanSingleLineDocComment()
Token Scanner::skipMultiLineComment()
{
size_t startPosition = m_source->position();
size_t startPosition = m_source.position();
while (!isSourcePastEndOfInput())
{
char prevChar = m_char;
@ -415,7 +400,7 @@ Token Scanner::skipMultiLineComment()
// multi-line comments are treated as whitespace.
if (prevChar == '*' && m_char == '/')
{
ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
if (unicodeDirectionError != ScannerError::NoError)
return setError(unicodeDirectionError);
@ -442,22 +427,22 @@ Token Scanner::scanMultiLineDocComment()
if (atEndOfLine())
{
skipWhitespace();
if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '*')
if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '*')
{ // it is unknown if this leads to the end of the comment
addCommentLiteralChar('*');
advance();
}
else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) != '/')
else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) != '/')
{ // skip first '*' in subsequent lines
m_char = m_source->advanceAndGet(1);
m_char = m_source.advanceAndGet(1);
if (atEndOfLine()) // ignores empty lines
continue;
if (charsAdded)
addCommentLiteralChar('\n'); // corresponds to the end of previous line
}
else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
{ // if after newline the comment ends, don't insert the newline
m_char = m_source->advanceAndGet(2);
m_char = m_source.advanceAndGet(2);
endFound = true;
break;
}
@ -465,9 +450,9 @@ Token Scanner::scanMultiLineDocComment()
addCommentLiteralChar('\n');
}
if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
{
m_char = m_source->advanceAndGet(2);
m_char = m_source.advanceAndGet(2);
endFound = true;
break;
}
@ -822,11 +807,11 @@ bool Scanner::isUnicodeLinebreak()
if (0x0a <= m_char && m_char <= 0x0d)
// line feed, vertical tab, form feed, carriage return
return true;
if (!m_source->isPastEndOfInput(1) && uint8_t(m_source->get(0)) == 0xc2 && uint8_t(m_source->get(1)) == 0x85)
if (!m_source.isPastEndOfInput(1) && uint8_t(m_source.get(0)) == 0xc2 && uint8_t(m_source.get(1)) == 0x85)
// NEL - U+0085, C2 85 in utf8
return true;
if (!m_source->isPastEndOfInput(2) && uint8_t(m_source->get(0)) == 0xe2 && uint8_t(m_source->get(1)) == 0x80 && (
uint8_t(m_source->get(2)) == 0xa8 || uint8_t(m_source->get(2)) == 0xa9
if (!m_source.isPastEndOfInput(2) && uint8_t(m_source.get(0)) == 0xe2 && uint8_t(m_source.get(1)) == 0x80 && (
uint8_t(m_source.get(2)) == 0xa8 || uint8_t(m_source.get(2)) == 0xa9
))
// LS - U+2028, E2 80 A8 in utf8
// PS - U+2029, E2 80 A9 in utf8
@ -836,7 +821,7 @@ bool Scanner::isUnicodeLinebreak()
Token Scanner::scanString(bool const _isUnicode)
{
size_t startPosition = m_source->position();
size_t startPosition = m_source.position();
char const quote = m_char;
advance(); // consume quote
LiteralScope literal(this, LITERAL_TYPE_STRING);
@ -867,7 +852,7 @@ Token Scanner::scanString(bool const _isUnicode)
if (_isUnicode)
{
ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
if (unicodeDirectionError != ScannerError::NoError)
return setError(unicodeDirectionError);
}
@ -921,7 +906,7 @@ void Scanner::scanDecimalDigits()
// May continue with decimal digit or underscore for grouping.
do
addLiteralCharAndAdvance();
while (!m_source->isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
while (!m_source.isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
// Defer further validation of underscore to SyntaxChecker.
}
@ -967,7 +952,7 @@ Token Scanner::scanNumber(char _charSeen)
scanDecimalDigits(); // optional
if (m_char == '.')
{
if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
{
// Assume the input may be a floating point number with leading '_' in fraction part.
// Recover by consuming it all but returning `Illegal` right away.
@ -975,7 +960,7 @@ Token Scanner::scanNumber(char _charSeen)
addLiteralCharAndAdvance(); // '_'
scanDecimalDigits();
}
if (m_source->isPastEndOfInput() || !isDecimalDigit(m_source->get(1)))
if (m_source.isPastEndOfInput() || !isDecimalDigit(m_source.get(1)))
{
// A '.' has to be followed by a number.
literal.complete();
@ -992,7 +977,7 @@ Token Scanner::scanNumber(char _charSeen)
solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
if (kind != DECIMAL)
return setError(ScannerError::IllegalExponent);
else if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
{
// Recover from wrongly placed underscore as delimiter in literal with scientific
// notation by consuming until the end.

26
liblangutil/Scanner.h
View File

@ -100,17 +100,13 @@ class Scanner
{
friend class LiteralScope;
public:
explicit Scanner(std::shared_ptr<CharStream> _source) { reset(std::move(_source)); }
explicit Scanner(CharStream _source = CharStream()) { reset(std::move(_source)); }
explicit Scanner(CharStream& _source):
m_source(_source),
m_sourceName{std::make_shared<std::string>(_source.name())}
{
reset();
}
std::string const& source() const noexcept { return m_source->source(); }
std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
std::shared_ptr<CharStream const> charStream() const noexcept { return m_source; }
/// Resets the scanner as if newly constructed with _source as input.
void reset(CharStream _source);
void reset(std::shared_ptr<CharStream> _source);
/// Resets scanner to the start of input.
void reset();
@ -201,8 +197,8 @@ private:
void addUnicodeAsUTF8(unsigned codepoint);
///@}
bool advance() { m_char = m_source->advanceAndGet(); return !m_source->isPastEndOfInput(); }
void rollback(size_t _amount) { m_char = m_source->rollback(_amount); }
bool advance() { m_char = m_source.advanceAndGet(); return !m_source.isPastEndOfInput(); }
void rollback(size_t _amount) { m_char = m_source.rollback(_amount); }
/// Rolls back to the start of the current token and re-runs the scanner.
void rescan();
@ -251,15 +247,15 @@ private:
bool isUnicodeLinebreak();
/// Return the current source position.
size_t sourcePos() const { return m_source->position(); }
bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
size_t sourcePos() const { return m_source.position(); }
bool isSourcePastEndOfInput() const { return m_source.isPastEndOfInput(); }
enum TokenIndex { Current, Next, NextNext };
TokenDesc m_skippedComments[3] = {}; // desc for the current, next and nextnext skipped comment
TokenDesc m_tokens[3] = {}; // desc for the current, next and nextnext token
std::shared_ptr<CharStream> m_source;
CharStream& m_source;
std::shared_ptr<std::string const> m_sourceName;
ScannerKind m_kind = ScannerKind::Solidity;

3
libsolidity/ast/ASTJsonImporter.cpp
View File

@ -958,7 +958,8 @@ Json::Value ASTJsonImporter::member(Json::Value const& _node, string const& _nam
Token ASTJsonImporter::scanSingleToken(Json::Value const& _node)
{
langutil::Scanner scanner{langutil::CharStream(_node.asString(), "")};
langutil::CharStream charStream(_node.asString(), "");
langutil::Scanner scanner{charStream};
astAssert(scanner.peekNextToken() == Token::EOS, "Token string is too long.");
return scanner.currentToken();
}

10
libsolidity/codegen/CompilerContext.cpp
View File

@ -434,14 +434,14 @@ void CompilerContext::appendInlineAssembly(
ErrorList errors;
ErrorReporter errorReporter(errors);
auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, _sourceName));
langutil::CharStream charStream(_assembly, _sourceName);
yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
optional<langutil::SourceLocation> locationOverride;
if (!_system)
locationOverride = m_asm->currentSourceLocation();
shared_ptr<yul::Block> parserResult =
yul::Parser(errorReporter, dialect, std::move(locationOverride))
.parse(scanner, false);
.parse(make_shared<langutil::Scanner>(charStream), false);
#ifdef SOL_OUTPUT_ASM
cout << yul::AsmPrinter(&dialect)(*parserResult) << endl;
#endif
@ -457,7 +457,7 @@ void CompilerContext::appendInlineAssembly(
for (auto const& error: errorReporter.errors())
// TODO if we have "locationOverride", it will be the wrong char stream,
// but we do not have access to the solidity scanner.
message += SourceReferenceFormatter::formatErrorInformation(*error, *scanner->charStream());
message += SourceReferenceFormatter::formatErrorInformation(*error, charStream);
message += "-------------------------------------------\n";
solAssert(false, message);
@ -491,8 +491,8 @@ void CompilerContext::appendInlineAssembly(
solAssert(m_generatedYulUtilityCode.empty(), "");
m_generatedYulUtilityCode = yul::AsmPrinter(dialect)(*obj.code);
string code = yul::AsmPrinter{dialect}(*obj.code);
scanner = make_shared<langutil::Scanner>(langutil::CharStream(m_generatedYulUtilityCode, _sourceName));
obj.code = yul::Parser(errorReporter, dialect).parse(scanner, false);
langutil::CharStream charStream(m_generatedYulUtilityCode, _sourceName);
obj.code = yul::Parser(errorReporter, dialect).parse(make_shared<Scanner>(charStream), false);
*obj.analysisInfo = yul::AsmAnalyzer::analyzeStrictAssertCorrect(dialect, obj);
}

34
libsolidity/interface/CompilerStack.cpp
View File

@ -313,7 +313,7 @@ void CompilerStack::setSources(StringMap _sources)
if (m_stackState != Empty)
BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Must set sources before parsing."));
for (auto source: _sources)
m_sources[source.first].scanner = make_shared<Scanner>(CharStream(/*content*/std::move(source.second), /*name*/source.first));
m_sources[source.first].charStream = make_unique<CharStream>(/*content*/std::move(source.second), /*name*/source.first);
m_stackState = SourcesSet;
}
@ -336,8 +336,7 @@ bool CompilerStack::parse()
{
string const& path = sourcesToParse[i];
Source& source = m_sources[path];
source.scanner->reset();
source.ast = parser.parse(source.scanner);
source.ast = parser.parse(*source.charStream);
if (!source.ast)
solAssert(!Error::containsOnlyWarnings(m_errorReporter.errors()), "Parser returned null but did not report error.");
else
@ -348,7 +347,7 @@ bool CompilerStack::parse()
{
string const& newPath = newSource.first;
string const& newContents = newSource.second;
m_sources[newPath].scanner = make_shared<Scanner>(CharStream(newContents, newPath));
m_sources[newPath].charStream = make_shared<CharStream>(newContents, newPath);
sourcesToParse.push_back(newPath);
}
}
@ -377,10 +376,11 @@ void CompilerStack::importASTs(map<string, Json::Value> const& _sources)
string const& path = src.first;
Source source;
source.ast = src.second;
string srcString = util::jsonCompactPrint(m_sourceJsons[src.first]);
ASTPointer<Scanner> scanner = make_shared<Scanner>(langutil::CharStream(srcString, src.first));
source.scanner = scanner;
m_sources[path] = source;
source.charStream = make_shared<CharStream>(
util::jsonCompactPrint(m_sourceJsons[src.first]),
src.first
);
m_sources[path] = move(source);
}
m_stackState = ParsedAndImported;
m_importedSources = true;
@ -754,7 +754,8 @@ Json::Value CompilerStack::generatedSources(string const& _contractName, bool _r
unsigned sourceIndex = sourceIndices()[sourceName];
ErrorList errors;
ErrorReporter errorReporter(errors);
auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(source, sourceName));
CharStream charStream(source, sourceName);
shared_ptr<Scanner> scanner = make_shared<Scanner>(charStream);
yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
shared_ptr<yul::Block> parserResult = yul::Parser{errorReporter, dialect}.parse(scanner, false);
solAssert(parserResult, "");
@ -1031,10 +1032,9 @@ CharStream const& CompilerStack::charStream(string const& _sourceName) const
if (m_stackState < SourcesSet)
BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("No sources set."));
solAssert(source(_sourceName).scanner, "");
solAssert(source(_sourceName).scanner->charStream(), "");
solAssert(source(_sourceName).charStream, "");
return *source(_sourceName).scanner->charStream();
return *source(_sourceName).charStream;
}
SourceUnit const& CompilerStack::ast(string const& _sourceName) const
@ -1079,21 +1079,21 @@ size_t CompilerStack::functionEntryPoint(
h256 const& CompilerStack::Source::keccak256() const
{
if (keccak256HashCached == h256{})
keccak256HashCached = util::keccak256(scanner->source());
keccak256HashCached = util::keccak256(charStream->source());
return keccak256HashCached;
}
h256 const& CompilerStack::Source::swarmHash() const
{
if (swarmHashCached == h256{})
swarmHashCached = util::bzzr1Hash(scanner->source());
swarmHashCached = util::bzzr1Hash(charStream->source());
return swarmHashCached;
}
string const& CompilerStack::Source::ipfsUrl() const
{
if (ipfsUrlCached.empty())
ipfsUrlCached = "dweb:/ipfs/" + util::ipfsHashBase58(scanner->source());
ipfsUrlCached = "dweb:/ipfs/" + util::ipfsHashBase58(charStream->source());
return ipfsUrlCached;
}
@ -1454,12 +1454,12 @@ string CompilerStack::createMetadata(Contract const& _contract) const
if (!referencedSources.count(s.first))
continue;
solAssert(s.second.scanner, "Scanner not available");
solAssert(s.second.charStream, "Character stream not available");
meta["sources"][s.first]["keccak256"] = "0x" + toHex(s.second.keccak256().asBytes());
if (optional<string> licenseString = s.second.ast->licenseString())
meta["sources"][s.first]["license"] = *licenseString;
if (m_metadataLiteralSources)
meta["sources"][s.first]["content"] = s.second.scanner->source();
meta["sources"][s.first]["content"] = s.second.charStream->source();
else
{
meta["sources"][s.first]["urls"] = Json::arrayValue;

3
libsolidity/interface/CompilerStack.h
View File

@ -57,7 +57,6 @@
namespace solidity::langutil
{
class Scanner;
class CharStream;
}
@ -344,7 +343,7 @@ private:
/// The state per source unit. Filled gradually during parsing.
struct Source
{
std::shared_ptr<langutil::Scanner> scanner;
std::shared_ptr<langutil::CharStream> charStream;
std::shared_ptr<SourceUnit> ast;
util::h256 mutable keccak256HashCached;
util::h256 mutable swarmHashCached;

11
libsolidity/parsing/Parser.cpp
View File

@ -81,13 +81,14 @@ private:
SourceLocation m_location;
};
ASTPointer<SourceUnit> Parser::parse(shared_ptr<Scanner> const& _scanner)
ASTPointer<SourceUnit> Parser::parse(CharStream& _charStream)
{
solAssert(!m_insideModifier, "");
try
{
m_recursionDepth = 0;
m_scanner = _scanner;
m_source = &_charStream;
m_scanner = make_shared<Scanner>(_charStream);
ASTNodeFactory nodeFactory(*this);
vector<ASTPointer<ASTNode>> nodes;
@ -2056,14 +2057,16 @@ bool Parser::variableDeclarationStart()
optional<string> Parser::findLicenseString(std::vector<ASTPointer<ASTNode>> const& _nodes)
{
solAssert(!!m_source, "");
// We circumvent the scanner here, because it skips non-docstring comments.
static regex const licenseRegex("SPDX-License-Identifier:\\s*([a-zA-Z0-9 ()+.-]+)");
// Search inside all parts of the source not covered by parsed nodes.
// This will leave e.g. "global comments".
string const& source = m_scanner->source();
using iter = decltype(source.begin());
using iter = std::string::const_iterator;
vector<pair<iter, iter>> sequencesToSearch;
string const& source = m_source->source();
sequencesToSearch.emplace_back(source.begin(), source.end());
for (ASTPointer<ASTNode> const& node: _nodes)
if (node->location().hasText())

5
libsolidity/parsing/Parser.h
View File

@ -29,7 +29,7 @@
namespace solidity::langutil
{
class Scanner;
class CharStream;
}
namespace solidity::frontend
@ -47,7 +47,7 @@ public:
m_evmVersion(_evmVersion)
{}
ASTPointer<SourceUnit> parse(std::shared_ptr<langutil::Scanner> const& _scanner);
ASTPointer<SourceUnit> parse(langutil::CharStream& _charStream);
private:
class ASTNodeFactory;
@ -211,6 +211,7 @@ private:
/// Creates an empty ParameterList at the current location (used if parameters can be omitted).
ASTPointer<ParameterList> createEmptyParameterList();
langutil::CharStream* m_source = nullptr;
/// Flag that signifies whether '_' is parsed as a PlaceholderStatement or a regular identifier.
bool m_insideModifier = false;
langutil::EVMVersion m_evmVersion;

6
libyul/AsmJsonImporter.cpp
View File

@ -166,7 +166,8 @@ Literal AsmJsonImporter::createLiteral(Json::Value const& _node)
if (kind == "number")
{
langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")};
langutil::CharStream charStream(lit.value.str(), "");
langutil::Scanner scanner{charStream};
lit.kind = LiteralKind::Number;
yulAssert(
scanner.currentToken() == Token::Number,
@ -175,7 +176,8 @@ Literal AsmJsonImporter::createLiteral(Json::Value const& _node)
}
else if (kind == "bool")
{
langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")};
langutil::CharStream charStream(lit.value.str(), "");
langutil::Scanner scanner{charStream};
lit.kind = LiteralKind::Boolean;
yulAssert(
scanner.currentToken() == Token::TrueLiteral ||

1
libyul/AsmParser.h
View File

@ -90,6 +90,7 @@ public:
/// @param _reuseScanner if true, do check for end of input after the `}`.
/// @returns an empty shared pointer on error.
std::unique_ptr<Block> parse(std::shared_ptr<langutil::Scanner> const& _scanner, bool _reuseScanner);
// TODO: pass CharStream here instead ^^^^^^
protected:
langutil::SourceLocation currentLocation() const override

3
libyul/ObjectParser.cpp
View File

@ -130,7 +130,8 @@ optional<ObjectParser::SourceNameMap> ObjectParser::tryParseSourceNameMapping()
solAssert(sm.size() == 2, "");
auto text = m_scanner->currentCommentLiteral().substr(static_cast<size_t>(sm.position() + sm.length()));
Scanner scanner(make_shared<CharStream>(text, ""));
CharStream charStream(text, "");
Scanner scanner(charStream);
if (scanner.currentToken() == Token::EOS)
return SourceNameMap{};
SourceNameMap sourceNames;

11
libyul/backends/wasm/EVMToEwasmTranslator.cpp
View File

@ -125,7 +125,7 @@ void EVMToEwasmTranslator::parsePolyfill()
{
ErrorList errors;
ErrorReporter errorReporter(errors);
shared_ptr<Scanner> scanner{make_shared<Scanner>(CharStream(
CharStream charStream(
"{" +
string(solidity::yul::wasm::polyfill::Arithmetic) +
string(solidity::yul::wasm::polyfill::Bitwise) +
@ -135,15 +135,18 @@ void EVMToEwasmTranslator::parsePolyfill()
string(solidity::yul::wasm::polyfill::Keccak) +
string(solidity::yul::wasm::polyfill::Logical) +
string(solidity::yul::wasm::polyfill::Memory) +
"}", ""))};
m_polyfill = Parser(errorReporter, WasmDialect::instance()).parse(scanner, false);
"}", "");
m_polyfill = Parser(errorReporter, WasmDialect::instance()).parse(
make_shared<Scanner>(charStream),
false
);
if (!errors.empty())
{
string message;
for (auto const& err: errors)
message += langutil::SourceReferenceFormatter::formatErrorInformation(
*err,
SingletonCharStreamProvider(*scanner->charStream())
SingletonCharStreamProvider(charStream)
);
yulAssert(false, message);
}

284
test/liblangutil/Scanner.cpp
View File

@ -34,13 +34,15 @@ BOOST_AUTO_TEST_SUITE(ScannerTest)
BOOST_AUTO_TEST_CASE(test_empty)
{
Scanner scanner(CharStream{});
CharStream stream{};
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(smoke_test)
{
Scanner scanner(CharStream("function break;765 \t \"string1\",'string2'\nidentifier1", ""));
CharStream stream("function break;765 \t \"string1\",'string2'\nidentifier1", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Break);
BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
@ -58,7 +60,8 @@ BOOST_AUTO_TEST_CASE(smoke_test)
BOOST_AUTO_TEST_CASE(assembly_assign)
{
Scanner scanner(CharStream("let a := 1", ""));
CharStream stream("let a := 1", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Let);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
@ -69,7 +72,8 @@ BOOST_AUTO_TEST_CASE(assembly_assign)
BOOST_AUTO_TEST_CASE(assembly_multiple_assign)
{
Scanner scanner(CharStream("let a, b, c := 1", ""));
CharStream stream("let a, b, c := 1", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Let);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
@ -89,14 +93,16 @@ BOOST_AUTO_TEST_CASE(string_printable)
// Escape \ and " (since we are quoting with ")
if (v == '\\' || v == '"')
lit = string{'\\'} + lit;
Scanner scanner(CharStream(" { \"" + lit + "\"", ""));
CharStream stream(" { \"" + lit + "\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), string{static_cast<char>(v)});
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
// Special case of unescaped " for strings quoted with '
Scanner scanner(CharStream(" { '\"'", ""));
CharStream stream(" { '\"'", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "\"");
@ -110,7 +116,8 @@ BOOST_AUTO_TEST_CASE(string_nonprintable)
if (v >= 0x20 && v <= 0x7e)
continue;
string lit{static_cast<char>(v)};
Scanner scanner(CharStream(" { \"" + lit + "\"", ""));
CharStream stream(" { \"" + lit + "\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
if (v == '\n' || v == '\v' || v == '\f' || v == '\r')
@ -123,7 +130,8 @@ BOOST_AUTO_TEST_CASE(string_nonprintable)
BOOST_AUTO_TEST_CASE(string_escapes)
{
Scanner scanner(CharStream(" { \"a\\x61\"", ""));
CharStream stream(" { \"a\\x61\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa");
@ -131,25 +139,46 @@ BOOST_AUTO_TEST_CASE(string_escapes)
BOOST_AUTO_TEST_CASE(string_escapes_all)
{
Scanner scanner(CharStream(" { \"a\\x61\\n\\r\\t\"", ""));
CharStream stream(" { \"a\\x61\\n\\r\\t\"", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa\n\r\t");
}
struct TestScanner
{
unique_ptr<CharStream> stream;
unique_ptr<Scanner> scanner;
explicit TestScanner(string _text) { reset(move(_text)); }
void reset(std::string _text)
{
stream = make_unique<CharStream>(move(_text), "");
scanner = make_unique<Scanner>(*stream);
}
decltype(auto) currentToken() { return scanner->currentToken(); }
decltype(auto) next() { return scanner->next(); }
decltype(auto) currentError() { return scanner->currentError(); }
decltype(auto) currentLiteral() { return scanner->currentLiteral(); }
decltype(auto) currentCommentLiteral() { return scanner->currentCommentLiteral(); }
decltype(auto) currentLocation() { return scanner->currentLocation(); }
};
BOOST_AUTO_TEST_CASE(string_escapes_legal_before_080)
{
Scanner scanner(CharStream(" { \"a\\b", ""));
TestScanner scanner(" { \"a\\b");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
scanner.reset(CharStream(" { \"a\\f", ""));
scanner.reset(" { \"a\\f");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
scanner.reset(CharStream(" { \"a\\v", ""));
scanner.reset(" { \"a\\v");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@ -158,7 +187,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_legal_before_080)
BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
{
Scanner scanner(CharStream(" { \"a\\x61\\x00abc\"", ""));
TestScanner scanner(" { \"a\\x61\\x00abc\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("aa\0abc", 6));
@ -166,7 +195,8 @@ BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
BOOST_AUTO_TEST_CASE(string_escape_illegal)
{
Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)", ""));
CharStream stream(" bla \"\\x6rf\" (illegalescape)", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@ -180,7 +210,7 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal)
BOOST_AUTO_TEST_CASE(hex_numbers)
{
Scanner scanner(CharStream("var x = 0x765432536763762734623472346;", ""));
TestScanner scanner("var x = 0x765432536763762734623472346;");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@ -188,34 +218,35 @@ BOOST_AUTO_TEST_CASE(hex_numbers)
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x765432536763762734623472346");
BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("0x1234", ""));
scanner.reset("0x1234");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234");
scanner.reset(CharStream("0X1234", ""));
scanner.reset("0X1234");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
}
BOOST_AUTO_TEST_CASE(octal_numbers)
{
Scanner scanner(CharStream("07", ""));
TestScanner scanner("07");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
scanner.reset(CharStream("007", ""));
scanner.reset("007");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
scanner.reset(CharStream("-07", ""));
scanner.reset("-07");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
scanner.reset(CharStream("-.07", ""));
scanner.reset("-.07");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
scanner.reset(CharStream("0", ""));
scanner.reset("0");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
scanner.reset(CharStream("0.1", ""));
scanner.reset("0.1");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
}
BOOST_AUTO_TEST_CASE(scientific_notation)
{
Scanner scanner(CharStream("var x = 2e10;", ""));
CharStream stream("var x = 2e10;", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@ -227,14 +258,14 @@ BOOST_AUTO_TEST_CASE(scientific_notation)
BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
{
Scanner scanner(CharStream("function .a(", ""));
TestScanner scanner("function .a(");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function .a(", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("function .a(");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -244,7 +275,7 @@ BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
{
Scanner scanner(CharStream("function a..a(", ""));
TestScanner scanner("function a..a(");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
@ -252,8 +283,8 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function a...a(", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("function a...a(");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@ -262,14 +293,14 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
{
Scanner scanner(CharStream("function a.(", ""));
TestScanner scanner("function a.(");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("function a.(", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("function a.(");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@ -278,19 +309,19 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
BOOST_AUTO_TEST_CASE(trailing_dot_in_numbers)
{
Scanner scanner(CharStream("2.5", ""));
TestScanner scanner("2.5");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("2.5e10", ""));
scanner.reset("2.5e10");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(".5", ""));
scanner.reset(".5");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(".5e10", ""));
scanner.reset(".5e10");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("2.", ""));
scanner.reset("2.");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -299,7 +330,8 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_numbers)
BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("_1.2", ""));
CharStream stream("_1.2", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -308,11 +340,11 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("1._2", ""));
TestScanner scanner("1._2");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("1._", ""));
scanner.reset("1._");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@ -320,7 +352,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("_1e2", ""));
CharStream stream("_1e2", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@ -328,7 +361,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("1e_2", ""));
CharStream stream("1e_2", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2");
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -336,7 +370,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
{
Scanner scanner(CharStream("0x_abc", ""));
CharStream stream("0x_abc", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -345,7 +380,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("12._1234_1234", ""));
CharStream stream("12._1234_1234", "");
Scanner scanner(stream);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@ -353,22 +389,22 @@ BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos)
{
// Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("0x123_", ""));
TestScanner scanner("0x123_");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("123_", ""));
scanner.reset("123_");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("12.34_", ""));
scanner.reset("12.34_");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(negative_numbers)
{
Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;", ""));
TestScanner scanner("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@ -395,7 +431,7 @@ BOOST_AUTO_TEST_CASE(negative_numbers)
BOOST_AUTO_TEST_CASE(locations)
{
Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment", ""));
TestScanner scanner("function_identifier has ; -0x743/*comment*/\n ident //comment");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLocation().start, 0);
BOOST_CHECK_EQUAL(scanner.currentLocation().end, 19);
@ -418,7 +454,7 @@ BOOST_AUTO_TEST_CASE(locations)
BOOST_AUTO_TEST_CASE(ambiguities)
{
// test scanning of some operators which need look-ahead
Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=", ""));
TestScanner scanner("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LessThanOrEqual);
BOOST_CHECK_EQUAL(scanner.next(), Token::LessThan);
BOOST_CHECK_EQUAL(scanner.next(), Token::Add);
@ -440,21 +476,21 @@ BOOST_AUTO_TEST_CASE(ambiguities)
BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin)
{
Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user", ""));
TestScanner scanner("/// Send $(value / 1000) chocolates to the user");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
}
BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed_begin)
{
Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/", ""));
TestScanner scanner("/** Send $(value / 1000) chocolates to the user*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
}
BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
{
Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user", ""));
TestScanner scanner("some other tokens /// Send $(value / 1000) chocolates to the user");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -464,9 +500,9 @@ BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
{
Scanner scanner(CharStream("some other tokens /**\n"
"* Send $(value / 1000) chocolates to the user\n"
"*/", ""));
TestScanner scanner("some other tokens /**\n"
"* Send $(value / 1000) chocolates to the user\n"
"*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -476,9 +512,9 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
{
Scanner scanner(CharStream("some other tokens /**\n"
" Send $(value / 1000) chocolates to the user\n"
"*/", ""));
TestScanner scanner("some other tokens /**\n"
" Send $(value / 1000) chocolates to the user\n"
"*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -488,9 +524,9 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
{
Scanner scanner(CharStream("some other tokens /** \t \r \n"
"\t \r * Send $(value / 1000) chocolates to the user\n"
"*/", ""));
TestScanner scanner("some other tokens /** \t \r \n"
"\t \r * Send $(value / 1000) chocolates to the user\n"
"*/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -500,37 +536,37 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
BOOST_AUTO_TEST_CASE(comment_before_eos)
{
Scanner scanner(CharStream("//", ""));
TestScanner scanner("//");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(documentation_comment_before_eos)
{
Scanner scanner(CharStream("///", ""));
TestScanner scanner("///");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(empty_multiline_comment)
{
Scanner scanner(CharStream("/**/", ""));
TestScanner scanner("/**/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(empty_multiline_documentation_comment_before_eos)
{
Scanner scanner(CharStream("/***/", ""));
TestScanner scanner("/***/");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
}
BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
{
Scanner scanner(CharStream("hello_world ///documentation comment \n"
"//simple comment \n"
"<<", ""));
TestScanner scanner("hello_world ///documentation comment \n"
"//simple comment \n"
"<<");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::SHL);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "documentation comment ");
@ -538,7 +574,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
BOOST_AUTO_TEST_CASE(ether_subdenominations)
{
Scanner scanner(CharStream("wei gwei ether", ""));
TestScanner scanner("wei gwei ether");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubWei);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubGwei);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubEther);
@ -546,7 +582,7 @@ BOOST_AUTO_TEST_CASE(ether_subdenominations)
BOOST_AUTO_TEST_CASE(time_subdenominations)
{
Scanner scanner(CharStream("seconds minutes hours days weeks years", ""));
TestScanner scanner("seconds minutes hours days weeks years");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubSecond);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubMinute);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubHour);
@ -557,7 +593,7 @@ BOOST_AUTO_TEST_CASE(time_subdenominations)
BOOST_AUTO_TEST_CASE(empty_comment)
{
Scanner scanner(CharStream("//\ncontract{}", ""));
TestScanner scanner("//\ncontract{}");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Contract);
BOOST_CHECK_EQUAL(scanner.next(), Token::LBrace);
@ -569,7 +605,7 @@ BOOST_AUTO_TEST_CASE(empty_comment)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
{
Scanner scanner(CharStream("{ \"\\u00DAnicode\"", ""));
TestScanner scanner("{ \"\\u00DAnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xC3\x9Anicode", 8));
@ -577,7 +613,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
{
Scanner scanner(CharStream("{ \"\\u007Fnicode\"", ""));
TestScanner scanner("{ \"\\u007Fnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x7Fnicode", 7));
@ -585,7 +621,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
{
Scanner scanner(CharStream("{ \"\\u07FFnicode\"", ""));
TestScanner scanner("{ \"\\u07FFnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xDF\xBFnicode", 8));
@ -593,7 +629,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
{
Scanner scanner(CharStream("{ \"\\uFFFFnicode\"", ""));
TestScanner scanner("{ \"\\uFFFFnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xEF\xBF\xBFnicode", 9));
@ -601,7 +637,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
{
Scanner scanner(CharStream("{ \"\\uFFnicode\"", ""));
TestScanner scanner("{ \"\\uFFnicode\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
}
@ -610,12 +646,12 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
BOOST_AUTO_TEST_CASE(unicode_prefix_only)
{
Scanner scanner(CharStream("{ unicode", ""));
TestScanner scanner("{ unicode");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ unicode", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ unicode");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "unicode");
@ -623,7 +659,7 @@ BOOST_AUTO_TEST_CASE(unicode_prefix_only)
BOOST_AUTO_TEST_CASE(unicode_invalid_space)
{
Scanner scanner(CharStream("{ unicode ", ""));
TestScanner scanner("{ unicode ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@ -631,12 +667,12 @@ BOOST_AUTO_TEST_CASE(unicode_invalid_space)
BOOST_AUTO_TEST_CASE(unicode_invalid_token)
{
Scanner scanner(CharStream("{ unicode test", ""));
TestScanner scanner("{ unicode test");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ unicode test", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ unicode test");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "unicode");
@ -646,7 +682,7 @@ BOOST_AUTO_TEST_CASE(unicode_invalid_token)
BOOST_AUTO_TEST_CASE(valid_unicode_literal)
{
Scanner scanner(CharStream("{ unicode\"Hello 😃\"", ""));
TestScanner scanner("{ unicode\"Hello 😃\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::UnicodeStringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("Hello \xf0\x9f\x98\x83", 10));
@ -655,7 +691,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_literal)
BOOST_AUTO_TEST_CASE(valid_nonprintable_in_unicode_literal)
{
// Non-printable characters are allowed in unicode strings...
Scanner scanner(CharStream("{ unicode\"Hello \007😃\"", ""));
TestScanner scanner("{ unicode\"Hello \007😃\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::UnicodeStringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("Hello \x07\xf0\x9f\x98\x83", 11));
@ -665,19 +701,19 @@ BOOST_AUTO_TEST_CASE(valid_nonprintable_in_unicode_literal)
BOOST_AUTO_TEST_CASE(hex_prefix_only)
{
Scanner scanner(CharStream("{ hex", ""));
TestScanner scanner("{ hex");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ hex", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ hex");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
}
BOOST_AUTO_TEST_CASE(hex_invalid_space)
{
Scanner scanner(CharStream("{ hex ", ""));
TestScanner scanner("{ hex ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@ -685,12 +721,12 @@ BOOST_AUTO_TEST_CASE(hex_invalid_space)
BOOST_AUTO_TEST_CASE(hex_invalid_token)
{
Scanner scanner(CharStream("{ hex test", ""));
TestScanner scanner("{ hex test");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
scanner.reset(CharStream("{ hex test", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("{ hex test");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@ -698,7 +734,7 @@ BOOST_AUTO_TEST_CASE(hex_invalid_token)
BOOST_AUTO_TEST_CASE(valid_hex_literal)
{
Scanner scanner(CharStream("{ hex\"00112233FF\"", ""));
TestScanner scanner("{ hex\"00112233FF\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::HexStringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x00\x11\x22\x33\xFF", 5));
@ -706,7 +742,7 @@ BOOST_AUTO_TEST_CASE(valid_hex_literal)
BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
{
Scanner scanner(CharStream("{ hex\"00112233F\"", ""));
TestScanner scanner("{ hex\"00112233F\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -714,7 +750,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
{
Scanner scanner(CharStream("{ hex\"00112233FF \"", ""));
TestScanner scanner("{ hex\"00112233FF \"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -722,7 +758,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
{
Scanner scanner(CharStream("{ hex\"00112233FF'", ""));
TestScanner scanner("{ hex\"00112233FF'");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -730,7 +766,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
{
Scanner scanner(CharStream("{ hex\"hello\"", ""));
TestScanner scanner("{ hex\"hello\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -741,7 +777,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
{
// This used to parse as "comment", "identifier"
Scanner scanner(CharStream("/** / x", ""));
TestScanner scanner("/** / x");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@ -749,14 +785,14 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos)
{
// This used to parse as "whitespace"
Scanner scanner(CharStream("/**", ""));
TestScanner scanner("/**");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(multiline_comment_at_eos)
{
Scanner scanner(CharStream("/*", ""));
TestScanner scanner("/*");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
@ -765,7 +801,7 @@ BOOST_AUTO_TEST_CASE(regular_line_break_in_single_line_comment)
{
for (auto const& nl: {"\r", "\n", "\r\n"})
{
Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
TestScanner scanner("// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@ -777,7 +813,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_comment)
{
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{
Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
TestScanner scanner("// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size() - 1; i++)
@ -792,7 +828,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_single_line_doc_comment)
{
for (auto const& nl: {"\r", "\n", "\r\n"})
{
Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
TestScanner scanner("/// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@ -806,7 +842,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_multiline_doc_comment)
// Any accepted non-LF is being canonicalized to LF.
for (auto const& nl : {"\r"s, "\n"s, "\r\n"s})
{
Scanner scanner{CharStream{"/// Hello" + nl + "/// World" + nl + "ident", ""}};
TestScanner scanner{"/// Hello" + nl + "/// World" + nl + "ident"};
auto const& lit = scanner.currentCommentLiteral();
BOOST_CHECK_EQUAL(lit, "Hello\n World");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Hello\n World");
@ -820,7 +856,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_doc_comment)
{
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{
Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
TestScanner scanner("/// abc " + string(nl) + " def ");
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size() - 1; i++)
@ -835,7 +871,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings)
{
for (auto const& nl: {"\r"s, "\n"s, "\r\n"s})
{
Scanner scanner(CharStream("\"abc " + nl + " def\"", ""));
TestScanner scanner("\"abc " + nl + " def\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@ -848,7 +884,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings)
{
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{
Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
TestScanner scanner("\"abc " + string(nl) + " def\"");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size(); i++)
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
@ -863,7 +899,7 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
{
// These are tokens which have a different meaning in Yul.
string keywords = "return byte bool address var in true false leave switch case default";
Scanner scanner(CharStream(keywords, ""));
TestScanner scanner(keywords);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Return);
BOOST_CHECK_EQUAL(scanner.next(), Token::Byte);
BOOST_CHECK_EQUAL(scanner.next(), Token::Bool);
@ -877,8 +913,8 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
BOOST_CHECK_EQUAL(scanner.next(), Token::Case);
BOOST_CHECK_EQUAL(scanner.next(), Token::Default);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(keywords, ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset(keywords);
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -896,28 +932,28 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
BOOST_AUTO_TEST_CASE(yul_keyword_like)
{
Scanner scanner(CharStream("leave.function", ""));
TestScanner scanner("leave.function");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("leave.function", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("leave.function");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
}
BOOST_AUTO_TEST_CASE(yul_identifier_with_dots)
{
Scanner scanner(CharStream("mystorage.slot := 1", ""));
TestScanner scanner("mystorage.slot := 1");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("mystorage.slot := 1", ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset("mystorage.slot := 1");
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
@ -927,7 +963,7 @@ BOOST_AUTO_TEST_CASE(yul_identifier_with_dots)
BOOST_AUTO_TEST_CASE(yul_function)
{
string sig = "function f(a, b) -> x, y";
Scanner scanner(CharStream(sig, ""));
TestScanner scanner(sig);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@ -940,8 +976,8 @@ BOOST_AUTO_TEST_CASE(yul_function)
BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(sig, ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset(sig);
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@ -959,7 +995,7 @@ BOOST_AUTO_TEST_CASE(yul_function)
BOOST_AUTO_TEST_CASE(yul_function_with_whitespace)
{
string sig = "function f (a, b) - > x, y";
Scanner scanner(CharStream(sig, ""));
TestScanner scanner(sig);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@ -973,8 +1009,8 @@ BOOST_AUTO_TEST_CASE(yul_function_with_whitespace)
BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(sig, ""));
scanner.setScannerMode(ScannerKind::Yul);
scanner.reset(sig);
scanner.scanner->setScannerMode(ScannerKind::Yul);
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);

5
test/libsolidity/Assembly.cpp
View File

@ -26,7 +26,8 @@
#include <liblangutil/SourceLocation.h>
#include <libevmasm/Assembly.h>
#include <liblangutil/Scanner.h>
#include <liblangutil/CharStream.h>
#include <libsolidity/parsing/Parser.h>
#include <libsolidity/analysis/DeclarationTypeChecker.h>
#include <libsolidity/analysis/NameAndTypeResolver.h>
@ -58,7 +59,7 @@ evmasm::AssemblyItems compileContract(std::shared_ptr<CharStream> _sourceCode)
ErrorReporter errorReporter(errors);
Parser parser(errorReporter, solidity::test::CommonOptions::get().evmVersion());
ASTPointer<SourceUnit> sourceUnit;
BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(_sourceCode)));
BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(*_sourceCode));
BOOST_CHECK(!!sourceUnit);
Scoper::assignScopes(*sourceUnit);

3
test/libsolidity/SemVerMatcher.cpp
View File

@ -43,7 +43,8 @@ namespace
SemVerMatchExpression parseExpression(string const& _input)
{
Scanner scanner{CharStream(_input, "")};
CharStream stream(_input, "");
Scanner scanner{stream};
vector<string> literals;
vector<Token> tokens;
while (scanner.currentToken() != Token::EOS)

5
test/libsolidity/SolidityExpressionCompiler.cpp
View File

@ -101,15 +101,14 @@ bytes compileFirstExpression(
)
{
string sourceCode = "pragma solidity >=0.0; // SPDX-License-Identifier: GPL-3\n" + _sourceCode;
CharStream stream(sourceCode, "");
ASTPointer<SourceUnit> sourceUnit;
try
{
ErrorList errors;
ErrorReporter errorReporter(errors);
sourceUnit = Parser(errorReporter, solidity::test::CommonOptions::get().evmVersion()).parse(
make_shared<Scanner>(CharStream(sourceCode, ""))
);
sourceUnit = Parser(errorReporter, solidity::test::CommonOptions::get().evmVersion()).parse(stream);
if (!sourceUnit)
return bytes();
}

3
test/libsolidity/SolidityParser.cpp
View File

@ -42,11 +42,12 @@ namespace
ASTPointer<ContractDefinition> parseText(std::string const& _source, ErrorList& _errors, bool errorRecovery = false)
{
ErrorReporter errorReporter(_errors);
auto charStream = CharStream(_source, "");
ASTPointer<SourceUnit> sourceUnit = Parser(
errorReporter,
solidity::test::CommonOptions::get().evmVersion(),
errorRecovery
).parse(std::make_shared<Scanner>(CharStream(_source, "")));
).parse(charStream);
if (!sourceUnit)
return ASTPointer<ContractDefinition>();
for (ASTPointer<ASTNode> const& node: sourceUnit->nodes())

3
test/libyul/Common.cpp
View File

@ -74,7 +74,8 @@ pair<shared_ptr<Object>, shared_ptr<yul::AsmAnalysisInfo>> yul::test::parse(
)
{
ErrorReporter errorReporter(_errors);
shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_source, ""));
CharStream stream(_source, "");
shared_ptr<Scanner> scanner = make_shared<Scanner>(stream);
shared_ptr<Object> parserResult = yul::ObjectParser(errorReporter, _dialect).parse(scanner, false);
if (!parserResult)
return {};

5
test/libyul/ObjectParser.cpp
View File

@ -23,6 +23,8 @@
#include <test/libsolidity/ErrorCheck.h>
#include <liblangutil/Scanner.h>
#include <libyul/AssemblyStack.h>
#include <libyul/backends/evm/EVMDialect.h>
@ -118,7 +120,8 @@ tuple<optional<ObjectParser::SourceNameMap>, ErrorList> tryGetSourceLocationMapp
ErrorReporter reporter(errors);
Dialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(EVMVersion::berlin());
ObjectParser objectParser{reporter, dialect};
objectParser.parse(make_shared<Scanner>(CharStream(move(source), "")), false);
CharStream stream(move(source), "");
objectParser.parse(make_shared<Scanner>(stream), false);
return {objectParser.sourceNameMapping(), std::move(errors)};
}

3
test/libyul/Parser.cpp
View File

@ -55,7 +55,8 @@ shared_ptr<Block> parse(string const& _source, Dialect const& _dialect, ErrorRep
{
try
{
auto scanner = make_shared<Scanner>(CharStream(_source, ""));
auto stream = CharStream(_source, "");
auto scanner = make_shared<Scanner>(stream);
map<unsigned, shared_ptr<string const>> indicesToSourceNames;
indicesToSourceNames[0] = make_shared<string const>("source0");
indicesToSourceNames[1] = make_shared<string const>("source1");

6
test/tools/yulopti.cpp
View File

@ -82,7 +82,7 @@ public:
{
SourceReferenceFormatter{
cerr,
SingletonCharStreamProvider(*m_scanner->charStream()),
SingletonCharStreamProvider(*m_charStream),
true,
false
}.printErrorInformation(m_errors);
@ -91,7 +91,8 @@ public:
bool parse(string const& _input)
{
ErrorReporter errorReporter(m_errors);
m_scanner = make_shared<Scanner>(CharStream(_input, ""));
m_charStream = make_shared<CharStream>(_input, "");
m_scanner = make_shared<Scanner>(*m_charStream);
m_ast = yul::Parser(errorReporter, m_dialect).parse(m_scanner, false);
if (!m_ast || !errorReporter.errors().empty())
{
@ -234,6 +235,7 @@ public:
private:
ErrorList m_errors;
shared_ptr<CharStream> m_charStream;
shared_ptr<Scanner> m_scanner;
shared_ptr<yul::Block> m_ast;
Dialect const& m_dialect{EVMDialect::strictAssemblyForEVMObjects(EVMVersion{})};

2
tools/yulPhaser/Program.cpp
View File

@ -120,7 +120,7 @@ variant<unique_ptr<Block>, ErrorList> Program::parseObject(Dialect const& _diale
{
ErrorList errors;
ErrorReporter errorReporter(errors);
auto scanner = make_shared<Scanner>(move(_source));
auto scanner = make_shared<Scanner>(_source);
ObjectParser parser(errorReporter, _dialect);
shared_ptr<Object> object = parser.parse(scanner, false);