cerc-io/solidity
16
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #5537 from ethereum/cp-SourceLocation-related-refactoring

Browse Source 
[1/3] SourceLocation related refactoring.
This commit is contained in:
chriseth 2018-11-30 23:45:31 +01:00 committed by GitHub
commit aaeb74f592
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 215 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
libevmasm/Assembly.cpp
View File

@ -114,10 +114,10 @@ namespace
string locationFromSources(StringMap const& _sourceCodes, SourceLocation const& _location) string locationFromSources(StringMap const& _sourceCodes, SourceLocation const& _location)
{ {
if (_location.isEmpty() || _sourceCodes.empty() || _location.start >= _location.end || _location.start < 0) if (_location.isEmpty() || !_location.source.get() || _sourceCodes.empty() || _location.start >= _location.end || _location.start < 0)
return ""; return "";
auto it = _sourceCodes.find(*_location.sourceName); auto it = _sourceCodes.find(_location.source->name());
if (it == _sourceCodes.end()) if (it == _sourceCodes.end())
return ""; return "";
@ -186,11 +186,11 @@ public:
void printLocation() void printLocation()
{ {
if (!m_location.sourceName && m_location.isEmpty()) if (!m_location.source && m_location.isEmpty())
return; return;
m_out << m_prefix << " /*"; m_out << m_prefix << " /*";
if (m_location.sourceName) if (m_location.source)
m_out << " \"" + *m_location.sourceName + "\""; m_out << " \"" + m_location.source->name() + "\"";
if (!m_location.isEmpty()) if (!m_location.isEmpty())
m_out << ":" << to_string(m_location.start) + ":" + to_string(m_location.end); m_out << ":" << to_string(m_location.start) + ":" + to_string(m_location.end);
m_out << " " << locationFromSources(m_sourceCodes, m_location); m_out << " " << locationFromSources(m_sourceCodes, m_location);

7
liblangutil/CharStream.h
View File

@ -68,7 +68,8 @@ class CharStream
{ {
public: public:
CharStream(): m_position(0) {} CharStream(): m_position(0) {}
explicit CharStream(std::string const& _source): m_source(_source), m_position(0) {} explicit CharStream(std::string const& _source, std::string const& name):
m_source(_source), m_name(name), m_position(0) {}
int position() const { return m_position; } int position() const { return m_position; }
bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); } bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); }
@ -79,7 +80,8 @@ public:
void reset() { m_position = 0; } void reset() { m_position = 0; }
std::string const& source() const { return m_source; } std::string const& source() const noexcept { return m_source; }
std::string const& name() const noexcept { return m_name; }
///@{ ///@{
///@name Error printing helper functions ///@name Error printing helper functions
@ -91,6 +93,7 @@ public:
private: private:
std::string m_source; std::string m_source;
std::string m_name;
size_t m_position; size_t m_position;
}; };

9
liblangutil/ParserBase.cpp
View File

@ -27,11 +27,6 @@
using namespace std; using namespace std;
using namespace langutil; using namespace langutil;
std::shared_ptr<string const> const& ParserBase::sourceName() const
{
return m_scanner->sourceName();
}
int ParserBase::position() const int ParserBase::position() const
{ {
return m_scanner->currentLocation().start; return m_scanner->currentLocation().start;
@ -105,10 +100,10 @@ void ParserBase::decreaseRecursionDepth()
void ParserBase::parserError(string const& _description) void ParserBase::parserError(string const& _description)
{ {
m_errorReporter.parserError(SourceLocation(position(), endPosition(), sourceName()), _description); m_errorReporter.parserError(SourceLocation(position(), endPosition(), source()), _description);
} }
void ParserBase::fatalParserError(string const& _description) void ParserBase::fatalParserError(string const& _description)
{ {
m_errorReporter.fatalParserError(SourceLocation(position(), endPosition(), sourceName()), _description); m_errorReporter.fatalParserError(SourceLocation(position(), endPosition(), source()), _description);
} }

3
liblangutil/ParserBase.h
View File

@ -23,6 +23,7 @@
#pragma once #pragma once
#include <liblangutil/Token.h> #include <liblangutil/Token.h>
#include <liblangutil/Scanner.h>
#include <memory> #include <memory>
#include <string> #include <string>
@ -37,7 +38,7 @@ class ParserBase
public: public:
explicit ParserBase(ErrorReporter& errorReporter): m_errorReporter(errorReporter) {} explicit ParserBase(ErrorReporter& errorReporter): m_errorReporter(errorReporter) {}
std::shared_ptr<std::string const> const& sourceName() const; std::shared_ptr<CharStream> source() const { return m_scanner->charStream(); }
protected: protected:
/// Utility class that creates an error and throws an exception if the /// Utility class that creates an error and throws an exception if the

54
liblangutil/Scanner.cpp
View File

@ -167,17 +167,23 @@ private:
}; // end of LiteralScope class }; // end of LiteralScope class
void Scanner::reset(CharStream _source, string _sourceName) void Scanner::reset(CharStream _source)
{ {
m_source = std::move(_source); m_source = make_shared<CharStream>(std::move(_source));
m_sourceName = make_shared<string const>(std::move(_sourceName)); reset();
}
void Scanner::reset(std::shared_ptr<CharStream> _source)
{
solAssert(_source.get() != nullptr, "You MUST provide a CharStream when resetting.");
m_source = _source;
reset(); reset();
} }
void Scanner::reset() void Scanner::reset()
{ {
m_source.reset(); m_source->reset();
m_char = m_source.get(); m_char = m_source->get();
skipWhitespace(); skipWhitespace();
scanToken(); scanToken();
next(); next();
@ -296,13 +302,13 @@ Token Scanner::scanSingleLineDocComment()
{ {
// check if next line is also a documentation comment // check if next line is also a documentation comment
skipWhitespace(); skipWhitespace();
if (!m_source.isPastEndOfInput(3) && if (!m_source->isPastEndOfInput(3) &&
m_source.get(0) == '/' && m_source->get(0) == '/' &&
m_source.get(1) == '/' && m_source->get(1) == '/' &&
m_source.get(2) == '/') m_source->get(2) == '/')
{ {
addCommentLiteralChar('\n'); addCommentLiteralChar('\n');
m_char = m_source.advanceAndGet(3); m_char = m_source->advanceAndGet(3);
} }
else else
break; // next line is not a documentation comment, we are done break; // next line is not a documentation comment, we are done
@ -355,20 +361,20 @@ Token Scanner::scanMultiLineDocComment()
if (isLineTerminator(m_char)) if (isLineTerminator(m_char))
{ {
skipWhitespace(); skipWhitespace();
if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '*') if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '*')
{ // it is unknown if this leads to the end of the comment { // it is unknown if this leads to the end of the comment
addCommentLiteralChar('*'); addCommentLiteralChar('*');
advance(); advance();
} }
else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) != '/') else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) != '/')
{ // skip first '*' in subsequent lines { // skip first '*' in subsequent lines
if (charsAdded) if (charsAdded)
addCommentLiteralChar('\n'); addCommentLiteralChar('\n');
m_char = m_source.advanceAndGet(2); m_char = m_source->advanceAndGet(2);
} }
else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/') else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
{ // if after newline the comment ends, don't insert the newline { // if after newline the comment ends, don't insert the newline
m_char = m_source.advanceAndGet(2); m_char = m_source->advanceAndGet(2);
endFound = true; endFound = true;
break; break;
} }
@ -376,9 +382,9 @@ Token Scanner::scanMultiLineDocComment()
addCommentLiteralChar('\n'); addCommentLiteralChar('\n');
} }
if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/') if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
{ {
m_char = m_source.advanceAndGet(2); m_char = m_source->advanceAndGet(2);
endFound = true; endFound = true;
break; break;
} }
@ -715,11 +721,11 @@ bool Scanner::isUnicodeLinebreak()
if (0x0a <= m_char && m_char <= 0x0d) if (0x0a <= m_char && m_char <= 0x0d)
// line feed, vertical tab, form feed, carriage return // line feed, vertical tab, form feed, carriage return
return true; return true;
else if (!m_source.isPastEndOfInput(1) && uint8_t(m_source.get(0)) == 0xc2 && uint8_t(m_source.get(1)) == 0x85) else if (!m_source->isPastEndOfInput(1) && uint8_t(m_source->get(0)) == 0xc2 && uint8_t(m_source->get(1)) == 0x85)
// NEL - U+0085, C2 85 in utf8 // NEL - U+0085, C2 85 in utf8
return true; return true;
else if (!m_source.isPastEndOfInput(2) && uint8_t(m_source.get(0)) == 0xe2 && uint8_t(m_source.get(1)) == 0x80 && ( else if (!m_source->isPastEndOfInput(2) && uint8_t(m_source->get(0)) == 0xe2 && uint8_t(m_source->get(1)) == 0x80 && (
uint8_t(m_source.get(2)) == 0xa8 || uint8_t(m_source.get(2)) == 0xa9 uint8_t(m_source->get(2)) == 0xa8 || uint8_t(m_source->get(2)) == 0xa9
)) ))
// LS - U+2028, E2 80 A8 in utf8 // LS - U+2028, E2 80 A8 in utf8
// PS - U+2029, E2 80 A9 in utf8 // PS - U+2029, E2 80 A9 in utf8
@ -783,7 +789,7 @@ void Scanner::scanDecimalDigits()
// May continue with decimal digit or underscore for grouping. // May continue with decimal digit or underscore for grouping.
do addLiteralCharAndAdvance(); do addLiteralCharAndAdvance();
while (!m_source.isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_')); while (!m_source->isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
// Defer further validation of underscore to SyntaxChecker. // Defer further validation of underscore to SyntaxChecker.
} }
@ -829,7 +835,7 @@ Token Scanner::scanNumber(char _charSeen)
scanDecimalDigits(); // optional scanDecimalDigits(); // optional
if (m_char == '.') if (m_char == '.')
{ {
if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_') if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
{ {
// Assume the input may be a floating point number with leading '_' in fraction part. // Assume the input may be a floating point number with leading '_' in fraction part.
// Recover by consuming it all but returning `Illegal` right away. // Recover by consuming it all but returning `Illegal` right away.
@ -837,7 +843,7 @@ Token Scanner::scanNumber(char _charSeen)
addLiteralCharAndAdvance(); // '_' addLiteralCharAndAdvance(); // '_'
scanDecimalDigits(); scanDecimalDigits();
} }
if (m_source.isPastEndOfInput() || !isDecimalDigit(m_source.get(1))) if (m_source->isPastEndOfInput() || !isDecimalDigit(m_source->get(1)))
{ {
// A '.' has to be followed by a number. // A '.' has to be followed by a number.
literal.complete(); literal.complete();
@ -854,7 +860,7 @@ Token Scanner::scanNumber(char _charSeen)
solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number"); solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
if (kind != DECIMAL) if (kind != DECIMAL)
return setError(ScannerError::IllegalExponent); return setError(ScannerError::IllegalExponent);
else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_') else if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
{ {
// Recover from wrongly placed underscore as delimiter in literal with scientific // Recover from wrongly placed underscore as delimiter in literal with scientific
// notation by consuming until the end. // notation by consuming until the end.

34
liblangutil/Scanner.h
View File

@ -90,12 +90,16 @@ class Scanner
{ {
friend class LiteralScope; friend class LiteralScope;
public: public:
explicit Scanner(CharStream _source = CharStream(), std::string _sourceName = "") { reset(std::move(_source), std::move(_sourceName)); } explicit Scanner(std::shared_ptr<CharStream> _source) { reset(std::move(_source)); }
explicit Scanner(CharStream _source = CharStream()) { reset(std::move(_source)); }
std::string source() const { return m_source.source(); } std::string const& source() const noexcept { return m_source->source(); }
/// Resets the scanner as if newly constructed with _source and _sourceName as input. std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
void reset(CharStream _source, std::string _sourceName);
/// Resets the scanner as if newly constructed with _source as input.
void reset(CharStream _source);
void reset(std::shared_ptr<CharStream> _source);
/// Resets scanner to the start of input. /// Resets scanner to the start of input.
void reset(); void reset();
@ -146,20 +150,17 @@ public:
std::string const& peekLiteral() const { return m_nextToken.literal; } std::string const& peekLiteral() const { return m_nextToken.literal; }
///@} ///@}
std::shared_ptr<std::string const> const& sourceName() const { return m_sourceName; }
///@{ ///@{
///@name Error printing helper functions ///@name Error printing helper functions
/// Functions that help pretty-printing parse errors /// Functions that help pretty-printing parse errors
/// Do only use in error cases, they are quite expensive. /// Do only use in error cases, they are quite expensive.
std::string lineAtPosition(int _position) const { return m_source.lineAtPosition(_position); } std::string lineAtPosition(int _position) const { return m_source->lineAtPosition(_position); }
std::tuple<int, int> translatePositionToLineColumn(int _position) const { return m_source.translatePositionToLineColumn(_position); } std::tuple<int, int> translatePositionToLineColumn(int _position) const { return m_source->translatePositionToLineColumn(_position); }
std::string sourceAt(SourceLocation const& _location) const std::string sourceAt(SourceLocation const& _location) const
{ {
solAssert(!_location.isEmpty(), ""); solAssert(!_location.isEmpty(), "");
solAssert(m_sourceName && _location.sourceName, ""); solAssert(m_source.get() == _location.source.get(), "CharStream memory locations must match.");
solAssert(*m_sourceName == *_location.sourceName, ""); return m_source->source().substr(_location.start, _location.end - _location.start);
return m_source.source().substr(_location.start, _location.end - _location.start);
} }
///@} ///@}
@ -188,8 +189,8 @@ private:
void addUnicodeAsUTF8(unsigned codepoint); void addUnicodeAsUTF8(unsigned codepoint);
///@} ///@}
bool advance() { m_char = m_source.advanceAndGet(); return !m_source.isPastEndOfInput(); } bool advance() { m_char = m_source->advanceAndGet(); return !m_source->isPastEndOfInput(); }
void rollback(int _amount) { m_char = m_source.rollback(_amount); } void rollback(int _amount) { m_char = m_source->rollback(_amount); }
inline Token selectErrorToken(ScannerError _err) { advance(); return setError(_err); } inline Token selectErrorToken(ScannerError _err) { advance(); return setError(_err); }
inline Token selectToken(Token _tok) { advance(); return _tok; } inline Token selectToken(Token _tok) { advance(); return _tok; }
@ -229,8 +230,8 @@ private:
bool isUnicodeLinebreak(); bool isUnicodeLinebreak();
/// Return the current source position. /// Return the current source position.
int sourcePos() const { return m_source.position(); } int sourcePos() const { return m_source->position(); }
bool isSourcePastEndOfInput() const { return m_source.isPastEndOfInput(); } bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
TokenDesc m_skippedComment; // desc for current skipped comment TokenDesc m_skippedComment; // desc for current skipped comment
TokenDesc m_nextSkippedComment; // desc for next skipped comment TokenDesc m_nextSkippedComment; // desc for next skipped comment
@ -238,8 +239,7 @@ private:
TokenDesc m_currentToken; // desc for current token (as returned by Next()) TokenDesc m_currentToken; // desc for current token (as returned by Next())
TokenDesc m_nextToken; // desc for next token (one token look-ahead) TokenDesc m_nextToken; // desc for next token (one token look-ahead)
CharStream m_source; std::shared_ptr<CharStream> m_source;
std::shared_ptr<std::string const> m_sourceName;
/// one character look-ahead, equals 0 at end of input /// one character look-ahead, equals 0 at end of input
char m_char; char m_char;

44
liblangutil/SourceLocation.h
View File

@ -23,6 +23,7 @@
#pragma once #pragma once
#include <libdevcore/Common.h> // defines noexcept macro for MSVC #include <libdevcore/Common.h> // defines noexcept macro for MSVC
#include <liblangutil/CharStream.h>
#include <memory> #include <memory>
#include <string> #include <string>
#include <ostream> #include <ostream>
@ -37,28 +38,13 @@ namespace langutil
*/ */
struct SourceLocation struct SourceLocation
{ {
SourceLocation(): start(-1), end(-1) { } SourceLocation(): start(-1), end(-1), source{nullptr} { }
SourceLocation(int _start, int _end, std::shared_ptr<std::string const> _sourceName): SourceLocation(int _start, int _end, std::shared_ptr<CharStream> _source):
start(_start), end(_end), sourceName(_sourceName) { } start(_start), end(_end), source{std::move(_source)} { }
SourceLocation(SourceLocation&& _other) noexcept:
start(_other.start),
end(_other.end),
sourceName(std::move(_other.sourceName))
{}
SourceLocation(SourceLocation const&) = default;
SourceLocation& operator=(SourceLocation const&) = default;
SourceLocation& operator=(SourceLocation&& _other) noexcept
{
start = _other.start;
end = _other.end;
sourceName = std::move(_other.sourceName);
return *this;
}
bool operator==(SourceLocation const& _other) const bool operator==(SourceLocation const& _other) const
{ {
return start == _other.start && end == _other.end && return source.get() == _other.source.get() && start == _other.start && end == _other.end;
((!sourceName && !_other.sourceName) || (sourceName && _other.sourceName && *sourceName == *_other.sourceName));
} }
bool operator!=(SourceLocation const& _other) const { return !operator==(_other); } bool operator!=(SourceLocation const& _other) const { return !operator==(_other); }
inline bool operator<(SourceLocation const& _other) const; inline bool operator<(SourceLocation const& _other) const;
@ -69,7 +55,7 @@ struct SourceLocation
int start; int start;
int end; int end;
std::shared_ptr<std::string const> sourceName; std::shared_ptr<CharStream> source;
}; };
/// Stream output for Location (used e.g. in boost exceptions). /// Stream output for Location (used e.g. in boost exceptions).
@ -77,27 +63,33 @@ inline std::ostream& operator<<(std::ostream& _out, SourceLocation const& _locat
{ {
if (_location.isEmpty()) if (_location.isEmpty())
return _out << "NO_LOCATION_SPECIFIED"; return _out << "NO_LOCATION_SPECIFIED";
return _out << *_location.sourceName << "[" << _location.start << "," << _location.end << ")";
if (_location.source)
_out << _location.source->name();
_out << "[" << _location.start << "," << _location.end << ")";
return _out;
} }
bool SourceLocation::operator<(SourceLocation const& _other) const bool SourceLocation::operator<(SourceLocation const& _other) const
{ {
if (!sourceName || !_other.sourceName) if (!source|| !_other.source)
return std::make_tuple(int(!!sourceName), start, end) < std::make_tuple(int(!!_other.sourceName), _other.start, _other.end); return std::make_tuple(int(!!source), start, end) < std::make_tuple(int(!!_other.source), _other.start, _other.end);
else else
return std::make_tuple(*sourceName, start, end) < std::make_tuple(*_other.sourceName, _other.start, _other.end); return std::make_tuple(source->name(), start, end) < std::make_tuple(_other.source->name(), _other.start, _other.end);
} }
bool SourceLocation::contains(SourceLocation const& _other) const bool SourceLocation::contains(SourceLocation const& _other) const
{ {
if (isEmpty() || _other.isEmpty() || ((!sourceName || !_other.sourceName || *sourceName != *_other.sourceName) && (sourceName || _other.sourceName))) if (isEmpty() || _other.isEmpty() || source.get() != _other.source.get())
return false; return false;
return start <= _other.start && _other.end <= end; return start <= _other.start && _other.end <= end;
} }
bool SourceLocation::intersects(SourceLocation const& _other) const bool SourceLocation::intersects(SourceLocation const& _other) const
{ {
if (isEmpty() || _other.isEmpty() || ((!sourceName || !_other.sourceName || *sourceName != *_other.sourceName) && (sourceName || _other.sourceName))) if (isEmpty() || _other.isEmpty() || source.get() != _other.source.get())
return false; return false;
return _other.start < end && start < _other.end; return _other.start < end && start < _other.end;
} }

10
liblangutil/SourceReferenceFormatter.cpp
View File

@ -30,9 +30,9 @@ using namespace langutil;
void SourceReferenceFormatter::printSourceLocation(SourceLocation const* _location) void SourceReferenceFormatter::printSourceLocation(SourceLocation const* _location)
{ {
if (!_location || !_location->sourceName) if (!_location || !_location->source)
return; // Nothing we can print here return; // Nothing we can print here
auto const& scanner = m_scannerFromSourceName(*_location->sourceName); auto const& scanner = m_scannerFromSourceName(_location->source->name());
int startLine; int startLine;
int startColumn; int startColumn;
tie(startLine, startColumn) = scanner.translatePositionToLineColumn(_location->start); tie(startLine, startColumn) = scanner.translatePositionToLineColumn(_location->start);
@ -89,13 +89,13 @@ void SourceReferenceFormatter::printSourceLocation(SourceLocation const* _locati
void SourceReferenceFormatter::printSourceName(SourceLocation const* _location) void SourceReferenceFormatter::printSourceName(SourceLocation const* _location)
{ {
if (!_location || !_location->sourceName) if (!_location || !_location->source)
return; // Nothing we can print here return; // Nothing we can print here
auto const& scanner = m_scannerFromSourceName(*_location->sourceName); auto const& scanner = m_scannerFromSourceName(_location->source->name());
int startLine; int startLine;
int startColumn; int startColumn;
tie(startLine, startColumn) = scanner.translatePositionToLineColumn(_location->start); tie(startLine, startColumn) = scanner.translatePositionToLineColumn(_location->start);
m_stream << *_location->sourceName << ":" << (startLine + 1) << ":" << (startColumn + 1) << ": "; m_stream << _location->source->name() << ":" << (startLine + 1) << ":" << (startColumn + 1) << ": ";
} }
void SourceReferenceFormatter::printExceptionInformation( void SourceReferenceFormatter::printExceptionInformation(

6
libsolidity/analysis/NameAndTypeResolver.cpp
View File

@ -492,9 +492,9 @@ bool DeclarationRegistrationHelper::registerDeclaration(
Declaration const* conflictingDeclaration = _container.conflictingDeclaration(_declaration, _name); Declaration const* conflictingDeclaration = _container.conflictingDeclaration(_declaration, _name);
solAssert(conflictingDeclaration, ""); solAssert(conflictingDeclaration, "");
bool const comparable = bool const comparable =
_errorLocation->sourceName && _errorLocation->source &&
conflictingDeclaration->location().sourceName && conflictingDeclaration->location().source &&
*_errorLocation->sourceName == *conflictingDeclaration->location().sourceName; _errorLocation->source->name() == conflictingDeclaration->location().source->name();
if (comparable && _errorLocation->start < conflictingDeclaration->location().start) if (comparable && _errorLocation->start < conflictingDeclaration->location().start)
{ {
firstDeclarationLocation = *_errorLocation; firstDeclarationLocation = *_errorLocation;

4
libsolidity/ast/ASTJsonConverter.cpp
View File

@ -122,8 +122,8 @@ void ASTJsonConverter::setJsonNode(
string ASTJsonConverter::sourceLocationToString(SourceLocation const& _location) const string ASTJsonConverter::sourceLocationToString(SourceLocation const& _location) const
{ {
int sourceIndex{-1}; int sourceIndex{-1};
if (_location.sourceName && m_sourceIndices.count(*_location.sourceName)) if (_location.source && m_sourceIndices.count(_location.source->name()))
sourceIndex = m_sourceIndices.at(*_location.sourceName); sourceIndex = m_sourceIndices.at(_location.source->name());
int length = -1; int length = -1;
if (_location.start >= 0 && _location.end >= 0) if (_location.start >= 0 && _location.end >= 0)
length = _location.end - _location.start; length = _location.end - _location.start;

2
libsolidity/codegen/CompilerContext.cpp
View File

@ -360,7 +360,7 @@ void CompilerContext::appendInlineAssembly(
ErrorList errors; ErrorList errors;
ErrorReporter errorReporter(errors); ErrorReporter errorReporter(errors);
auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly), "--CODEGEN--"); auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, "--CODEGEN--"));
auto parserResult = yul::Parser(errorReporter, yul::AsmFlavour::Strict).parse(scanner, false); auto parserResult = yul::Parser(errorReporter, yul::AsmFlavour::Strict).parse(scanner, false);
#ifdef SOL_OUTPUT_ASM #ifdef SOL_OUTPUT_ASM
cout << yul::AsmPrinter()(*parserResult) << endl; cout << yul::AsmPrinter()(*parserResult) << endl;

2
libsolidity/interface/AssemblyStack.cpp
View File

@ -69,7 +69,7 @@ bool AssemblyStack::parseAndAnalyze(std::string const& _sourceName, std::string
{ {
m_errors.clear(); m_errors.clear();
m_analysisSuccessful = false; m_analysisSuccessful = false;
m_scanner = make_shared<Scanner>(CharStream(_source), _sourceName); m_scanner = make_shared<Scanner>(CharStream(_source, _sourceName));
m_parserResult = yul::ObjectParser(m_errorReporter, languageToAsmFlavour(m_language)).parse(m_scanner, false); m_parserResult = yul::ObjectParser(m_errorReporter, languageToAsmFlavour(m_language)).parse(m_scanner, false);
if (!m_errorReporter.errors().empty()) if (!m_errorReporter.errors().empty())
return false; return false;

12
libsolidity/interface/CompilerStack.cpp
View File

@ -126,7 +126,7 @@ bool CompilerStack::addSource(string const& _name, string const& _content, bool
{ {
bool existed = m_sources.count(_name) != 0; bool existed = m_sources.count(_name) != 0;
reset(true); reset(true);
m_sources[_name].scanner = make_shared<Scanner>(CharStream(_content), _name); m_sources[_name].scanner = make_shared<Scanner>(CharStream(_content, _name));
m_sources[_name].isLibrary = _isLibrary; m_sources[_name].isLibrary = _isLibrary;
m_stackState = SourcesSet; m_stackState = SourcesSet;
return existed; return existed;
@ -161,7 +161,7 @@ bool CompilerStack::parse()
{ {
string const& newPath = newSource.first; string const& newPath = newSource.first;
string const& newContents = newSource.second; string const& newContents = newSource.second;
m_sources[newPath].scanner = make_shared<Scanner>(CharStream(newContents), newPath); m_sources[newPath].scanner = make_shared<Scanner>(CharStream(newContents, newPath));
sourcesToParse.push_back(newPath); sourcesToParse.push_back(newPath);
} }
} }
@ -612,8 +612,8 @@ tuple<int, int, int, int> CompilerStack::positionFromSourceLocation(SourceLocati
int startColumn; int startColumn;
int endLine; int endLine;
int endColumn; int endColumn;
tie(startLine, startColumn) = scanner(*_sourceLocation.sourceName).translatePositionToLineColumn(_sourceLocation.start); tie(startLine, startColumn) = scanner(_sourceLocation.source->name()).translatePositionToLineColumn(_sourceLocation.start);
tie(endLine, endColumn) = scanner(*_sourceLocation.sourceName).translatePositionToLineColumn(_sourceLocation.end); tie(endLine, endColumn) = scanner(_sourceLocation.source->name()).translatePositionToLineColumn(_sourceLocation.end);
return make_tuple(++startLine, ++startColumn, ++endLine, ++endColumn); return make_tuple(++startLine, ++startColumn, ++endLine, ++endColumn);
} }
@ -936,8 +936,8 @@ string CompilerStack::computeSourceMapping(eth::AssemblyItems const& _items) con
SourceLocation const& location = item.location(); SourceLocation const& location = item.location();
int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1; int length = location.start != -1 && location.end != -1 ? location.end - location.start : -1;
int sourceIndex = int sourceIndex =
location.sourceName && sourceIndicesMap.count(*location.sourceName) ? location.source && sourceIndicesMap.count(location.source->name()) ?
sourceIndicesMap.at(*location.sourceName) : sourceIndicesMap.at(location.source->name()) :
-1; -1;
char jump = '-'; char jump = '-';
if (item.getJumpType() == eth::AssemblyItem::JumpType::IntoFunction) if (item.getJumpType() == eth::AssemblyItem::JumpType::IntoFunction)

4
libsolidity/interface/StandardCompiler.cpp
View File

@ -85,9 +85,9 @@ Json::Value formatErrorWithException(
message = _message; message = _message;
Json::Value sourceLocation; Json::Value sourceLocation;
if (location && location->sourceName) if (location && location->source && location->source->name() != "")
{ {
sourceLocation["file"] = *location->sourceName; sourceLocation["file"] = location->source->name();
sourceLocation["start"] = location->start; sourceLocation["start"] = location->start;
sourceLocation["end"] = location->end; sourceLocation["end"] = location->end;
} }

4
libsolidity/parsing/Parser.cpp
View File

@ -42,7 +42,7 @@ class Parser::ASTNodeFactory
{ {
public: public:
explicit ASTNodeFactory(Parser const& _parser): explicit ASTNodeFactory(Parser const& _parser):
m_parser(_parser), m_location(_parser.position(), -1, _parser.sourceName()) {} m_parser(_parser), m_location(_parser.position(), -1, _parser.source()) {}
ASTNodeFactory(Parser const& _parser, ASTPointer<ASTNode> const& _childNode): ASTNodeFactory(Parser const& _parser, ASTPointer<ASTNode> const& _childNode):
m_parser(_parser), m_location(_childNode->location()) {} m_parser(_parser), m_location(_childNode->location()) {}
@ -55,7 +55,7 @@ public:
template <class NodeType, typename... Args> template <class NodeType, typename... Args>
ASTPointer<NodeType> createNode(Args&& ... _args) ASTPointer<NodeType> createNode(Args&& ... _args)
{ {
solAssert(m_location.sourceName, ""); solAssert(m_location.source, "");
if (m_location.end < 0) if (m_location.end < 0)
markEndPosition(); markEndPosition();
return make_shared<NodeType>(m_location, std::forward<Args>(_args)...); return make_shared<NodeType>(m_location, std::forward<Args>(_args)...);

6
libyul/AsmParser.h
View File

@ -56,11 +56,11 @@ protected:
r.location.start = position(); r.location.start = position();
r.location.end = endPosition(); r.location.end = endPosition();
} }
if (!r.location.sourceName) if (!r.location.source)
r.location.sourceName = sourceName(); r.location.source = m_scanner->charStream();
return r; return r;
} }
langutil::SourceLocation location() const { return {position(), endPosition(), sourceName()}; } langutil::SourceLocation location() const { return {position(), endPosition(), m_scanner->charStream()}; }
Block parseBlock(); Block parseBlock();
Statement parseStatement(); Statement parseStatement();

6
test/libevmasm/Assembler.cpp
View File

@ -55,10 +55,12 @@ BOOST_AUTO_TEST_SUITE(Assembler)
BOOST_AUTO_TEST_CASE(all_assembly_items) BOOST_AUTO_TEST_CASE(all_assembly_items)
{ {
Assembly _assembly; Assembly _assembly;
_assembly.setSourceLocation(SourceLocation(1, 3, make_shared<string>("root.asm"))); auto root_asm = make_shared<CharStream>("", "root.asm");
_assembly.setSourceLocation(SourceLocation(1, 3, root_asm));
Assembly _subAsm; Assembly _subAsm;
_subAsm.setSourceLocation(SourceLocation(6, 8, make_shared<string>("sub.asm"))); auto sub_asm = make_shared<CharStream>("", "sub.asm");
_subAsm.setSourceLocation(SourceLocation(6, 8, sub_asm));
_subAsm.append(Instruction::INVALID); _subAsm.append(Instruction::INVALID);
shared_ptr<Assembly> _subAsmPtr = make_shared<Assembly>(_subAsm); shared_ptr<Assembly> _subAsmPtr = make_shared<Assembly>(_subAsm);

2
test/libevmasm/Optimiser.cpp
View File

@ -53,7 +53,7 @@ namespace
// add dummy locations to each item so that we can check that they are not deleted // add dummy locations to each item so that we can check that they are not deleted
AssemblyItems input = _input; AssemblyItems input = _input;
for (AssemblyItem& item: input) for (AssemblyItem& item: input)
item.setLocation(SourceLocation(1, 3, make_shared<string>(""))); item.setLocation(SourceLocation(1, 3, nullptr));
return input; return input;
} }

14
test/liblangutil/SourceLocation.cpp
View File

@ -33,12 +33,16 @@ BOOST_AUTO_TEST_SUITE(SourceLocationTest)
BOOST_AUTO_TEST_CASE(test_fail) BOOST_AUTO_TEST_CASE(test_fail)
{ {
auto const source = std::make_shared<CharStream>("", "source");
auto const sourceA = std::make_shared<CharStream>("", "sourceA");
auto const sourceB = std::make_shared<CharStream>("", "sourceB");
BOOST_CHECK(SourceLocation() == SourceLocation()); BOOST_CHECK(SourceLocation() == SourceLocation());
BOOST_CHECK(SourceLocation(0, 3, std::make_shared<std::string>("sourceA")) != SourceLocation(0, 3, std::make_shared<std::string>("sourceB"))); BOOST_CHECK(SourceLocation(0, 3, sourceA) != SourceLocation(0, 3, sourceB));
BOOST_CHECK(SourceLocation(0, 3, std::make_shared<std::string>("source")) == SourceLocation(0, 3, std::make_shared<std::string>("source"))); BOOST_CHECK(SourceLocation(0, 3, source) == SourceLocation(0, 3, source));
BOOST_CHECK(SourceLocation(3, 7, std::make_shared<std::string>("source")).contains(SourceLocation(4, 6, std::make_shared<std::string>("source")))); BOOST_CHECK(SourceLocation(3, 7, source).contains(SourceLocation(4, 6, source)));
BOOST_CHECK(!SourceLocation(3, 7, std::make_shared<std::string>("sourceA")).contains(SourceLocation(4, 6, std::make_shared<std::string>("sourceB")))); BOOST_CHECK(!SourceLocation(3, 7, sourceA).contains(SourceLocation(4, 6, sourceB)));
BOOST_CHECK(SourceLocation(3, 7, std::make_shared<std::string>("sourceA")) < SourceLocation(4, 6, std::make_shared<std::string>("sourceB"))); BOOST_CHECK(SourceLocation(3, 7, sourceA) < SourceLocation(4, 6, sourceB));
} }
BOOST_AUTO_TEST_SUITE_END() BOOST_AUTO_TEST_SUITE_END()

42
test/libsolidity/Assembly.cpp
View File

@ -52,13 +52,13 @@ namespace test
namespace namespace
{ {
eth::AssemblyItems compileContract(string const& _sourceCode) eth::AssemblyItems compileContract(std::shared_ptr<CharStream> _sourceCode)
{ {
ErrorList errors; ErrorList errors;
ErrorReporter errorReporter(errors); ErrorReporter errorReporter(errors);
Parser parser(errorReporter); Parser parser(errorReporter);
ASTPointer<SourceUnit> sourceUnit; ASTPointer<SourceUnit> sourceUnit;
BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(CharStream(_sourceCode)))); BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(_sourceCode)));
BOOST_CHECK(!!sourceUnit); BOOST_CHECK(!!sourceUnit);
map<ASTNode const*, shared_ptr<DeclarationContainer>> scopes; map<ASTNode const*, shared_ptr<DeclarationContainer>> scopes;
@ -104,7 +104,7 @@ void printAssemblyLocations(AssemblyItems const& _items)
", " << ", " <<
_loc.end << _loc.end <<
", make_shared<string>(\"" << ", make_shared<string>(\"" <<
*_loc.sourceName << _loc.source->name() <<
"\"))) +" << endl; "\"))) +" << endl;
}; };
@ -135,7 +135,8 @@ void checkAssemblyLocations(AssemblyItems const& _items, vector<SourceLocation>
BOOST_CHECK_EQUAL(_items.size(), _locations.size()); BOOST_CHECK_EQUAL(_items.size(), _locations.size());
for (size_t i = 0; i < min(_items.size(), _locations.size()); ++i) for (size_t i = 0; i < min(_items.size(), _locations.size()); ++i)
{ {
if (_items[i].location() != _locations[i]) if (_items[i].location().start != _locations[i].start ||
_items[i].location().end != _locations[i].end)
{ {
BOOST_CHECK_MESSAGE(false, "Location mismatch for item " + to_string(i) + ". Found the following locations:"); BOOST_CHECK_MESSAGE(false, "Location mismatch for item " + to_string(i) + ". Found the following locations:");
printAssemblyLocations(_items); printAssemblyLocations(_items);
@ -151,29 +152,32 @@ BOOST_AUTO_TEST_SUITE(Assembly)
BOOST_AUTO_TEST_CASE(location_test) BOOST_AUTO_TEST_CASE(location_test)
{ {
char const* sourceCode = R"( auto sourceCode = make_shared<CharStream>(R"(
contract test { contract test {
function f() public returns (uint256 a) { function f() public returns (uint256 a) {
return 16; return 16;
} }
} }
)"; )", "");
AssemblyItems items = compileContract(sourceCode); AssemblyItems items = compileContract(sourceCode);
bool hasShifts = dev::test::Options::get().evmVersion().hasBitwiseShifting(); bool hasShifts = dev::test::Options::get().evmVersion().hasBitwiseShifting();
auto codegenCharStream = make_shared<CharStream>("", "--CODEGEN--");
vector<SourceLocation> locations = vector<SourceLocation> locations =
vector<SourceLocation>(hasShifts ? 21 : 22, SourceLocation(2, 82, make_shared<string>(""))) + vector<SourceLocation>(hasShifts ? 21 : 22, SourceLocation(2, 82, sourceCode)) +
vector<SourceLocation>(2, SourceLocation(20, 79, make_shared<string>(""))) + vector<SourceLocation>(2, SourceLocation(20, 79, sourceCode)) +
vector<SourceLocation>(1, SourceLocation(8, 17, make_shared<string>("--CODEGEN--"))) + vector<SourceLocation>(1, SourceLocation(8, 17, codegenCharStream)) +
vector<SourceLocation>(3, SourceLocation(5, 7, make_shared<string>("--CODEGEN--"))) + vector<SourceLocation>(3, SourceLocation(5, 7, codegenCharStream)) +
vector<SourceLocation>(1, SourceLocation(30, 31, make_shared<string>("--CODEGEN--"))) + vector<SourceLocation>(1, SourceLocation(30, 31, codegenCharStream)) +
vector<SourceLocation>(1, SourceLocation(27, 28, make_shared<string>("--CODEGEN--"))) + vector<SourceLocation>(1, SourceLocation(27, 28, codegenCharStream)) +
vector<SourceLocation>(1, SourceLocation(20, 32, make_shared<string>("--CODEGEN--"))) + vector<SourceLocation>(1, SourceLocation(20, 32, codegenCharStream)) +
vector<SourceLocation>(1, SourceLocation(5, 7, make_shared<string>("--CODEGEN--"))) + vector<SourceLocation>(1, SourceLocation(5, 7, codegenCharStream)) +
vector<SourceLocation>(24, SourceLocation(20, 79, make_shared<string>(""))) + vector<SourceLocation>(24, SourceLocation(20, 79, sourceCode)) +
vector<SourceLocation>(1, SourceLocation(49, 58, make_shared<string>(""))) + vector<SourceLocation>(1, SourceLocation(49, 58, sourceCode)) +
vector<SourceLocation>(1, SourceLocation(72, 74, make_shared<string>(""))) + vector<SourceLocation>(1, SourceLocation(72, 74, sourceCode)) +
vector<SourceLocation>(2, SourceLocation(65, 74, make_shared<string>(""))) + vector<SourceLocation>(2, SourceLocation(65, 74, sourceCode)) +
vector<SourceLocation>(2, SourceLocation(20, 79, make_shared<string>(""))); vector<SourceLocation>(2, SourceLocation(20, 79, sourceCode));
checkAssemblyLocations(items, locations); checkAssemblyLocations(items, locations);
} }

2
test/libsolidity/SemVerMatcher.cpp
View File

@ -41,7 +41,7 @@ BOOST_AUTO_TEST_SUITE(SemVerMatcher)
SemVerMatchExpression parseExpression(string const& _input) SemVerMatchExpression parseExpression(string const& _input)
{ {
Scanner scanner{CharStream(_input)}; Scanner scanner{CharStream(_input, "")};
vector<string> literals; vector<string> literals;
vector<Token> tokens; vector<Token> tokens;
while (scanner.currentToken() != Token::EOS) while (scanner.currentToken() != Token::EOS)

2
test/libsolidity/SolidityExpressionCompiler.cpp
View File

@ -101,7 +101,7 @@ bytes compileFirstExpression(
{ {
ErrorList errors; ErrorList errors;
ErrorReporter errorReporter(errors); ErrorReporter errorReporter(errors);
sourceUnit = Parser(errorReporter).parse(make_shared<Scanner>(CharStream(_sourceCode))); sourceUnit = Parser(errorReporter).parse(make_shared<Scanner>(CharStream(_sourceCode, "")));
if (!sourceUnit) if (!sourceUnit)
return bytes(); return bytes();
} }

2
test/libsolidity/SolidityParser.cpp
View File

@ -43,7 +43,7 @@ namespace
ASTPointer<ContractDefinition> parseText(std::string const& _source, ErrorList& _errors) ASTPointer<ContractDefinition> parseText(std::string const& _source, ErrorList& _errors)
{ {
ErrorReporter errorReporter(_errors); ErrorReporter errorReporter(_errors);
ASTPointer<SourceUnit> sourceUnit = Parser(errorReporter).parse(std::make_shared<Scanner>(CharStream(_source))); ASTPointer<SourceUnit> sourceUnit = Parser(errorReporter).parse(std::make_shared<Scanner>(CharStream(_source, "")));
if (!sourceUnit) if (!sourceUnit)
return ASTPointer<ContractDefinition>(); return ASTPointer<ContractDefinition>();
for (ASTPointer<ASTNode> const& node: sourceUnit->nodes()) for (ASTPointer<ASTNode> const& node: sourceUnit->nodes())

134
test/libsolidity/SolidityScanner.cpp
View File

@ -37,13 +37,13 @@ BOOST_AUTO_TEST_SUITE(SolidityScanner)
BOOST_AUTO_TEST_CASE(test_empty) BOOST_AUTO_TEST_CASE(test_empty)
{ {
Scanner scanner(CharStream("")); Scanner scanner(CharStream{});
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
} }
BOOST_AUTO_TEST_CASE(smoke_test) BOOST_AUTO_TEST_CASE(smoke_test)
{ {
Scanner scanner(CharStream("function break;765 \t \"string1\",'string2'\nidentifier1")); Scanner scanner(CharStream("function break;765 \t \"string1\",'string2'\nidentifier1", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
BOOST_CHECK_EQUAL(scanner.next(), Token::Break); BOOST_CHECK_EQUAL(scanner.next(), Token::Break);
BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon); BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(smoke_test)
BOOST_AUTO_TEST_CASE(string_escapes) BOOST_AUTO_TEST_CASE(string_escapes)
{ {
Scanner scanner(CharStream(" { \"a\\x61\"")); Scanner scanner(CharStream(" { \"a\\x61\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa");
@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(string_escapes)
BOOST_AUTO_TEST_CASE(string_escapes_all) BOOST_AUTO_TEST_CASE(string_escapes_all)
{ {
Scanner scanner(CharStream(" { \"a\\x61\\b\\f\\n\\r\\t\\v\"")); Scanner scanner(CharStream(" { \"a\\x61\\b\\f\\n\\r\\t\\v\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa\b\f\n\r\t\v"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa\b\f\n\r\t\v");
@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_all)
BOOST_AUTO_TEST_CASE(string_escapes_with_zero) BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
{ {
Scanner scanner(CharStream(" { \"a\\x61\\x00abc\"")); Scanner scanner(CharStream(" { \"a\\x61\\x00abc\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("aa\0abc", 6)); BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("aa\0abc", 6));
@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
BOOST_AUTO_TEST_CASE(string_escape_illegal) BOOST_AUTO_TEST_CASE(string_escape_illegal)
{ {
Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)")); Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence); BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@ -99,7 +99,7 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal)
BOOST_AUTO_TEST_CASE(hex_numbers) BOOST_AUTO_TEST_CASE(hex_numbers)
{ {
Scanner scanner(CharStream("var x = 0x765432536763762734623472346;")); Scanner scanner(CharStream("var x = 0x765432536763762734623472346;", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign); BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@ -107,34 +107,34 @@ BOOST_AUTO_TEST_CASE(hex_numbers)
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x765432536763762734623472346"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x765432536763762734623472346");
BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon); BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("0x1234"), ""); scanner.reset(CharStream("0x1234", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234");
scanner.reset(CharStream("0X1234"), ""); scanner.reset(CharStream("0X1234", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
} }
BOOST_AUTO_TEST_CASE(octal_numbers) BOOST_AUTO_TEST_CASE(octal_numbers)
{ {
Scanner scanner(CharStream("07")); Scanner scanner(CharStream("07", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
scanner.reset(CharStream("007"), ""); scanner.reset(CharStream("007", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
scanner.reset(CharStream("-07"), ""); scanner.reset(CharStream("-07", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
scanner.reset(CharStream("-.07"), ""); scanner.reset(CharStream("-.07", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number); BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
scanner.reset(CharStream("0"), ""); scanner.reset(CharStream("0", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
scanner.reset(CharStream("0.1"), ""); scanner.reset(CharStream("0.1", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
} }
BOOST_AUTO_TEST_CASE(scientific_notation) BOOST_AUTO_TEST_CASE(scientific_notation)
{ {
Scanner scanner(CharStream("var x = 2e10;")); Scanner scanner(CharStream("var x = 2e10;", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign); BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@ -146,19 +146,19 @@ BOOST_AUTO_TEST_CASE(scientific_notation)
BOOST_AUTO_TEST_CASE(trailing_dot) BOOST_AUTO_TEST_CASE(trailing_dot)
{ {
Scanner scanner(CharStream("2.5")); Scanner scanner(CharStream("2.5", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("2.5e10"), ""); scanner.reset(CharStream("2.5e10", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(".5"), ""); scanner.reset(CharStream(".5", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream(".5e10"), ""); scanner.reset(CharStream(".5e10", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("2."), ""); scanner.reset(CharStream("2.", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::Period); BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -167,7 +167,7 @@ BOOST_AUTO_TEST_CASE(trailing_dot)
BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier) BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
{ {
// Actual error is cought by SyntaxChecker. // Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("_1.2")); Scanner scanner(CharStream("_1.2", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Number); BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -176,11 +176,11 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal) BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
{ {
// Actual error is cought by SyntaxChecker. // Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("1._2")); Scanner scanner(CharStream("1._2", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("1._"), ""); scanner.reset(CharStream("1._", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
@ -188,7 +188,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier) BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
{ {
// Actual error is cought by SyntaxChecker. // Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("_1e2")); Scanner scanner(CharStream("_1e2", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
@ -196,7 +196,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal) BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
{ {
// Actual error is cought by SyntaxChecker. // Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("1e_2")); Scanner scanner(CharStream("1e_2", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2");
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -204,7 +204,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal) BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
{ {
Scanner scanner(CharStream("0x_abc")); Scanner scanner(CharStream("0x_abc", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@ -213,7 +213,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front) BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
{ {
// Actual error is cought by SyntaxChecker. // Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("12._1234_1234")); Scanner scanner(CharStream("12._1234_1234", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
@ -221,22 +221,22 @@ BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos) BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos)
{ {
// Actual error is cought by SyntaxChecker. // Actual error is cought by SyntaxChecker.
Scanner scanner(CharStream("0x123_")); Scanner scanner(CharStream("0x123_", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("123_"), ""); scanner.reset(CharStream("123_", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
scanner.reset(CharStream("12.34_"), ""); scanner.reset(CharStream("12.34_", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
BOOST_AUTO_TEST_CASE(negative_numbers) BOOST_AUTO_TEST_CASE(negative_numbers)
{ {
Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;")); Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Assign); BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@ -263,7 +263,7 @@ BOOST_AUTO_TEST_CASE(negative_numbers)
BOOST_AUTO_TEST_CASE(locations) BOOST_AUTO_TEST_CASE(locations)
{ {
Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment")); Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLocation().start, 0); BOOST_CHECK_EQUAL(scanner.currentLocation().start, 0);
BOOST_CHECK_EQUAL(scanner.currentLocation().end, 19); BOOST_CHECK_EQUAL(scanner.currentLocation().end, 19);
@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(locations)
BOOST_AUTO_TEST_CASE(ambiguities) BOOST_AUTO_TEST_CASE(ambiguities)
{ {
// test scanning of some operators which need look-ahead // test scanning of some operators which need look-ahead
Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=")); Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LessThanOrEqual); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LessThanOrEqual);
BOOST_CHECK_EQUAL(scanner.next(), Token::LessThan); BOOST_CHECK_EQUAL(scanner.next(), Token::LessThan);
BOOST_CHECK_EQUAL(scanner.next(), Token::Add); BOOST_CHECK_EQUAL(scanner.next(), Token::Add);
@ -308,21 +308,21 @@ BOOST_AUTO_TEST_CASE(ambiguities)
BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin) BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin)
{ {
Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user")); Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user"); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
} }
BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed_begin) BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed_begin)
{ {
Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/")); Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user"); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
} }
BOOST_AUTO_TEST_CASE(documentation_comments_parsed) BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
{ {
Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user")); Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -334,7 +334,7 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
{ {
Scanner scanner(CharStream("some other tokens /**\n" Scanner scanner(CharStream("some other tokens /**\n"
"* Send $(value / 1000) chocolates to the user\n" "* Send $(value / 1000) chocolates to the user\n"
"*/")); "*/", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -346,7 +346,7 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
{ {
Scanner scanner(CharStream("some other tokens /**\n" Scanner scanner(CharStream("some other tokens /**\n"
" Send $(value / 1000) chocolates to the user\n" " Send $(value / 1000) chocolates to the user\n"
"*/")); "*/", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -358,7 +358,7 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
{ {
Scanner scanner(CharStream("some other tokens /** \t \r \n" Scanner scanner(CharStream("some other tokens /** \t \r \n"
"\t \r * Send $(value / 1000) chocolates to the user\n" "\t \r * Send $(value / 1000) chocolates to the user\n"
"*/")); "*/", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@ -368,28 +368,28 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
BOOST_AUTO_TEST_CASE(comment_before_eos) BOOST_AUTO_TEST_CASE(comment_before_eos)
{ {
Scanner scanner(CharStream("//")); Scanner scanner(CharStream("//", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
} }
BOOST_AUTO_TEST_CASE(documentation_comment_before_eos) BOOST_AUTO_TEST_CASE(documentation_comment_before_eos)
{ {
Scanner scanner(CharStream("///")); Scanner scanner(CharStream("///", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
} }
BOOST_AUTO_TEST_CASE(empty_multiline_comment) BOOST_AUTO_TEST_CASE(empty_multiline_comment)
{ {
Scanner scanner(CharStream("/**/")); Scanner scanner(CharStream("/**/", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
} }
BOOST_AUTO_TEST_CASE(empty_multiline_documentation_comment_before_eos) BOOST_AUTO_TEST_CASE(empty_multiline_documentation_comment_before_eos)
{ {
Scanner scanner(CharStream("/***/")); Scanner scanner(CharStream("/***/", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
} }
@ -398,7 +398,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
{ {
Scanner scanner(CharStream("hello_world ///documentation comment \n" Scanner scanner(CharStream("hello_world ///documentation comment \n"
"//simple comment \n" "//simple comment \n"
"<<")); "<<", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.next(), Token::SHL); BOOST_CHECK_EQUAL(scanner.next(), Token::SHL);
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "documentation comment "); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "documentation comment ");
@ -406,7 +406,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
BOOST_AUTO_TEST_CASE(ether_subdenominations) BOOST_AUTO_TEST_CASE(ether_subdenominations)
{ {
Scanner scanner(CharStream("wei szabo finney ether")); Scanner scanner(CharStream("wei szabo finney ether", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubWei); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubWei);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubSzabo); BOOST_CHECK_EQUAL(scanner.next(), Token::SubSzabo);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubFinney); BOOST_CHECK_EQUAL(scanner.next(), Token::SubFinney);
@ -415,7 +415,7 @@ BOOST_AUTO_TEST_CASE(ether_subdenominations)
BOOST_AUTO_TEST_CASE(time_subdenominations) BOOST_AUTO_TEST_CASE(time_subdenominations)
{ {
Scanner scanner(CharStream("seconds minutes hours days weeks years")); Scanner scanner(CharStream("seconds minutes hours days weeks years", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubSecond); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubSecond);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubMinute); BOOST_CHECK_EQUAL(scanner.next(), Token::SubMinute);
BOOST_CHECK_EQUAL(scanner.next(), Token::SubHour); BOOST_CHECK_EQUAL(scanner.next(), Token::SubHour);
@ -426,7 +426,7 @@ BOOST_AUTO_TEST_CASE(time_subdenominations)
BOOST_AUTO_TEST_CASE(empty_comment) BOOST_AUTO_TEST_CASE(empty_comment)
{ {
Scanner scanner(CharStream("//\ncontract{}")); Scanner scanner(CharStream("//\ncontract{}", ""));
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Contract); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Contract);
BOOST_CHECK_EQUAL(scanner.next(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.next(), Token::LBrace);
@ -436,7 +436,7 @@ BOOST_AUTO_TEST_CASE(empty_comment)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape) BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
{ {
Scanner scanner(CharStream("{ \"\\u00DAnicode\"")); Scanner scanner(CharStream("{ \"\\u00DAnicode\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xC3\x9Anicode", 8)); BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xC3\x9Anicode", 8));
@ -444,7 +444,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f) BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
{ {
Scanner scanner(CharStream("{ \"\\u007Fnicode\"")); Scanner scanner(CharStream("{ \"\\u007Fnicode\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x7Fnicode", 7)); BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x7Fnicode", 7));
@ -452,7 +452,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff) BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
{ {
Scanner scanner(CharStream("{ \"\\u07FFnicode\"")); Scanner scanner(CharStream("{ \"\\u07FFnicode\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xDF\xBFnicode", 8)); BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xDF\xBFnicode", 8));
@ -460,7 +460,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff) BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
{ {
Scanner scanner(CharStream("{ \"\\uFFFFnicode\"")); Scanner scanner(CharStream("{ \"\\uFFFFnicode\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xEF\xBF\xBFnicode", 9)); BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xEF\xBF\xBFnicode", 9));
@ -468,7 +468,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape) BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
{ {
Scanner scanner(CharStream("{ \"\\uFFnicode\"")); Scanner scanner(CharStream("{ \"\\uFFnicode\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
} }
@ -477,7 +477,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
BOOST_AUTO_TEST_CASE(valid_hex_literal) BOOST_AUTO_TEST_CASE(valid_hex_literal)
{ {
Scanner scanner(CharStream("{ hex\"00112233FF\"")); Scanner scanner(CharStream("{ hex\"00112233FF\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral); BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x00\x11\x22\x33\xFF", 5)); BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x00\x11\x22\x33\xFF", 5));
@ -485,7 +485,7 @@ BOOST_AUTO_TEST_CASE(valid_hex_literal)
BOOST_AUTO_TEST_CASE(invalid_short_hex_literal) BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
{ {
Scanner scanner(CharStream("{ hex\"00112233F\"")); Scanner scanner(CharStream("{ hex\"00112233F\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString); BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -493,7 +493,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space) BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
{ {
Scanner scanner(CharStream("{ hex\"00112233FF \"")); Scanner scanner(CharStream("{ hex\"00112233FF \"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString); BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -501,7 +501,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes) BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
{ {
Scanner scanner(CharStream("{ hex\"00112233FF'")); Scanner scanner(CharStream("{ hex\"00112233FF'", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString); BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -509,7 +509,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string) BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
{ {
Scanner scanner(CharStream("{ hex\"hello\"")); Scanner scanner(CharStream("{ hex\"hello\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString); BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@ -520,7 +520,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close) BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
{ {
// This used to parse as "comment", "identifier" // This used to parse as "comment", "identifier"
Scanner scanner(CharStream("/** / x")); Scanner scanner(CharStream("/** / x", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
@ -528,14 +528,14 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos) BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos)
{ {
// This used to parse as "whitespace" // This used to parse as "whitespace"
Scanner scanner(CharStream("/**")); Scanner scanner(CharStream("/**", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
BOOST_AUTO_TEST_CASE(multiline_comment_at_eos) BOOST_AUTO_TEST_CASE(multiline_comment_at_eos)
{ {
Scanner scanner(CharStream("/*")); Scanner scanner(CharStream("/*", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
} }
@ -544,7 +544,7 @@ BOOST_AUTO_TEST_CASE(regular_line_break_in_single_line_comment)
{ {
for (auto const& nl: {"\r", "\n"}) for (auto const& nl: {"\r", "\n"})
{ {
Scanner scanner(CharStream("// abc " + string(nl) + " def ")); Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@ -556,7 +556,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_comment)
{ {
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"}) for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{ {
Scanner scanner(CharStream("// abc " + string(nl) + " def ")); Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), ""); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size() - 1; i++) for (size_t i = 0; i < string(nl).size() - 1; i++)
@ -571,7 +571,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_single_line_doc_comment)
{ {
for (auto const& nl: {"\r", "\n"}) for (auto const& nl: {"\r", "\n"})
{ {
Scanner scanner(CharStream("/// abc " + string(nl) + " def ")); Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc "); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@ -583,7 +583,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_doc_comment)
{ {
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"}) for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{ {
Scanner scanner(CharStream("/// abc " + string(nl) + " def ")); Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc "); BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size() - 1; i++) for (size_t i = 0; i < string(nl).size() - 1; i++)
@ -598,7 +598,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings)
{ {
for (auto const& nl: {"\n", "\r"}) for (auto const& nl: {"\n", "\r"})
{ {
Scanner scanner(CharStream("\"abc " + string(nl) + " def\"")); Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def"); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@ -611,7 +611,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings)
{ {
for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"}) for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
{ {
Scanner scanner(CharStream("\"abc " + string(nl) + " def\"")); Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
for (size_t i = 0; i < string(nl).size(); i++) for (size_t i = 0; i < string(nl).size(); i++)
BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);

2
test/libyul/Common.cpp
View File

@ -57,7 +57,7 @@ pair<shared_ptr<Block>, shared_ptr<yul::AsmAnalysisInfo>> yul::test::parse(strin
auto flavour = _yul ? yul::AsmFlavour::Yul : yul::AsmFlavour::Strict; auto flavour = _yul ? yul::AsmFlavour::Yul : yul::AsmFlavour::Strict;
ErrorList errors; ErrorList errors;
ErrorReporter errorReporter(errors); ErrorReporter errorReporter(errors);
auto scanner = make_shared<Scanner>(CharStream(_source), ""); auto scanner = make_shared<Scanner>(CharStream(_source, ""));
auto parserResult = yul::Parser(errorReporter, flavour).parse(scanner, false); auto parserResult = yul::Parser(errorReporter, flavour).parse(scanner, false);
if (parserResult) if (parserResult)
{ {

2
test/libyul/Parser.cpp
View File

@ -51,7 +51,7 @@ bool parse(string const& _source, ErrorReporter& errorReporter)
{ {
try try
{ {
auto scanner = make_shared<Scanner>(CharStream(_source)); auto scanner = make_shared<Scanner>(CharStream(_source, ""));
auto parserResult = yul::Parser(errorReporter, yul::AsmFlavour::Yul).parse(scanner, false); auto parserResult = yul::Parser(errorReporter, yul::AsmFlavour::Yul).parse(scanner, false);
if (parserResult) if (parserResult)
{ {

2
test/libyul/YulOptimizerTest.cpp
View File

@ -259,7 +259,7 @@ bool YulOptimizerTest::parse(ostream& _stream, string const& _linePrefix, bool c
yul::AsmFlavour flavour = m_yul ? yul::AsmFlavour::Yul : yul::AsmFlavour::Strict; yul::AsmFlavour flavour = m_yul ? yul::AsmFlavour::Yul : yul::AsmFlavour::Strict;
ErrorList errors; ErrorList errors;
ErrorReporter errorReporter(errors); ErrorReporter errorReporter(errors);
shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(m_source), ""); shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(m_source, ""));
m_ast = yul::Parser(errorReporter, flavour).parse(scanner, false); m_ast = yul::Parser(errorReporter, flavour).parse(scanner, false);
if (!m_ast || !errorReporter.errors().empty()) if (!m_ast || !errorReporter.errors().empty())
{ {

2
test/tools/yulopti.cpp
View File

@ -81,7 +81,7 @@ public:
bool parse(string const& _input) bool parse(string const& _input)
{ {
ErrorReporter errorReporter(m_errors); ErrorReporter errorReporter(m_errors);
shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_input), ""); shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_input, ""));
m_ast = yul::Parser(errorReporter, yul::AsmFlavour::Strict).parse(scanner, false); m_ast = yul::Parser(errorReporter, yul::AsmFlavour::Strict).parse(scanner, false);
if (!m_ast || !errorReporter.errors().empty()) if (!m_ast || !errorReporter.errors().empty())
{ {