Merge pull request #11658 from ethereum/removeScannerFromCompilerStack

Remove scanner from compiler stack

liblangutil/ParserBase.h

@@ -24,7 +24,6 @@
 #pragma once
 
 #include <liblangutil/Token.h>
-#include <liblangutil/Scanner.h>
 #include <memory>
 #include <string>
 
@@ -50,8 +49,6 @@ public:
 
 	virtual ~ParserBase() = default;
 
-	std::shared_ptr<CharStream> source() const { return m_scanner->charStream(); }
-
 protected:
 	/// Utility class that creates an error and throws an exception if the
 	/// recursion depth is too deep.

liblangutil/Scanner.cpp

@@ -135,26 +135,11 @@ private:
 	bool m_complete;
 };
 
-void Scanner::reset(CharStream _source)
-{
-	m_source = make_shared<CharStream>(std::move(_source));
-	m_sourceName = make_shared<string>(m_source->name());
-	reset();
-}
-
-void Scanner::reset(shared_ptr<CharStream> _source)
-{
-	solAssert(_source.get() != nullptr, "You MUST provide a CharStream when resetting.");
-	m_source = std::move(_source);
-	m_sourceName = make_shared<string>(m_source->name());
-	reset();
-}
-
 void Scanner::reset()
 {
-	m_source->reset();
+	m_source.reset();
 	m_kind = ScannerKind::Solidity;
-	m_char = m_source->get();
+	m_char = m_source.get();
 	skipWhitespace();
 	next();
 	next();
@@ -163,7 +148,7 @@ void Scanner::reset()
 
 void Scanner::setPosition(size_t _offset)
 {
-	m_char = m_source->setPosition(_offset);
+	m_char = m_source.setPosition(_offset);
 	scanToken();
 	next();
 	next();
@@ -229,7 +214,7 @@ void Scanner::rescan()
 		rollbackTo = static_cast<size_t>(m_tokens[Current].location.start);
 	else
 		rollbackTo = static_cast<size_t>(m_skippedComments[Current].location.start);
-	m_char = m_source->rollback(m_source->position() - rollbackTo);
+	m_char = m_source.rollback(m_source.position() - rollbackTo);
 	next();
 	next();
 	next();
@@ -324,12 +309,12 @@ Token Scanner::skipSingleLineComment()
 {
 	// Line terminator is not part of the comment. If it is a
 	// non-ascii line terminator, it will result in a parser error.
-	size_t startPosition = m_source->position();
+	size_t startPosition = m_source.position();
 	while (!isUnicodeLinebreak())
 		if (!advance())
 			break;
 
-	ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
+	ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
 	if (unicodeDirectionError != ScannerError::NoError)
 		return setError(unicodeDirectionError);
 
@@ -362,28 +347,28 @@ bool Scanner::tryScanEndOfLine()
 size_t Scanner::scanSingleLineDocComment()
 {
 	LiteralScope literal(this, LITERAL_TYPE_COMMENT);
-	size_t endPosition = m_source->position();
+	size_t endPosition = m_source.position();
 
 	skipWhitespaceExceptUnicodeLinebreak();
 
 	while (!isSourcePastEndOfInput())
 	{
-		endPosition = m_source->position();
+		endPosition = m_source.position();
 		if (tryScanEndOfLine())
 		{
 			// Check if next line is also a single-line comment.
 			// If any whitespaces were skipped, use source position before.
 			if (!skipWhitespaceExceptUnicodeLinebreak())
-				endPosition = m_source->position();
+				endPosition = m_source.position();
 
-			if (!m_source->isPastEndOfInput(3) &&
+			if (!m_source.isPastEndOfInput(3) &&
-				m_source->get(0) == '/' &&
+				m_source.get(0) == '/' &&
-				m_source->get(1) == '/' &&
+				m_source.get(1) == '/' &&
-				m_source->get(2) == '/')
+				m_source.get(2) == '/')
 			{
-				if (!m_source->isPastEndOfInput(4) && m_source->get(3) == '/')
+				if (!m_source.isPastEndOfInput(4) && m_source.get(3) == '/')
 					break; // "////" is not a documentation comment
-				m_char = m_source->advanceAndGet(3);
+				m_char = m_source.advanceAndGet(3);
 				if (atEndOfLine())
 					continue;
 				addCommentLiteralChar('\n');
@@ -404,7 +389,7 @@ size_t Scanner::scanSingleLineDocComment()
 
 Token Scanner::skipMultiLineComment()
 {
-	size_t startPosition = m_source->position();
+	size_t startPosition = m_source.position();
 	while (!isSourcePastEndOfInput())
 	{
 		char prevChar = m_char;
@@ -415,7 +400,7 @@ Token Scanner::skipMultiLineComment()
 		// multi-line comments are treated as whitespace.
 		if (prevChar == '*' && m_char == '/')
 		{
-			ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
+			ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
 			if (unicodeDirectionError != ScannerError::NoError)
 				return setError(unicodeDirectionError);
 
@@ -442,22 +427,22 @@ Token Scanner::scanMultiLineDocComment()
 		if (atEndOfLine())
 		{
 			skipWhitespace();
-			if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '*')
+			if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '*')
 			{ // it is unknown if this leads to the end of the comment
 				addCommentLiteralChar('*');
 				advance();
 			}
-			else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) != '/')
+			else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) != '/')
 			{ // skip first '*' in subsequent lines
-				m_char = m_source->advanceAndGet(1);
+				m_char = m_source.advanceAndGet(1);
 				if (atEndOfLine()) // ignores empty lines
 					continue;
 				if (charsAdded)
 					addCommentLiteralChar('\n'); // corresponds to the end of previous line
 			}
-			else if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
+			else if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
 			{ // if after newline the comment ends, don't insert the newline
-				m_char = m_source->advanceAndGet(2);
+				m_char = m_source.advanceAndGet(2);
 				endFound = true;
 				break;
 			}
@@ -465,9 +450,9 @@ Token Scanner::scanMultiLineDocComment()
 				addCommentLiteralChar('\n');
 		}
 
-		if (!m_source->isPastEndOfInput(1) && m_source->get(0) == '*' && m_source->get(1) == '/')
+		if (!m_source.isPastEndOfInput(1) && m_source.get(0) == '*' && m_source.get(1) == '/')
 		{
-			m_char = m_source->advanceAndGet(2);
+			m_char = m_source.advanceAndGet(2);
 			endFound = true;
 			break;
 		}
@@ -822,11 +807,11 @@ bool Scanner::isUnicodeLinebreak()
 	if (0x0a <= m_char && m_char <= 0x0d)
 		// line feed, vertical tab, form feed, carriage return
 		return true;
-	if (!m_source->isPastEndOfInput(1) && uint8_t(m_source->get(0)) == 0xc2 && uint8_t(m_source->get(1)) == 0x85)
+	if (!m_source.isPastEndOfInput(1) && uint8_t(m_source.get(0)) == 0xc2 && uint8_t(m_source.get(1)) == 0x85)
 		// NEL - U+0085, C2 85 in utf8
 		return true;
-	if (!m_source->isPastEndOfInput(2) && uint8_t(m_source->get(0)) == 0xe2 && uint8_t(m_source->get(1)) == 0x80 && (
+	if (!m_source.isPastEndOfInput(2) && uint8_t(m_source.get(0)) == 0xe2 && uint8_t(m_source.get(1)) == 0x80 && (
-		uint8_t(m_source->get(2)) == 0xa8 || uint8_t(m_source->get(2)) == 0xa9
+		uint8_t(m_source.get(2)) == 0xa8 || uint8_t(m_source.get(2)) == 0xa9
 	))
 		// LS - U+2028, E2 80 A8  in utf8
 		// PS - U+2029, E2 80 A9  in utf8
@@ -836,7 +821,7 @@ bool Scanner::isUnicodeLinebreak()
 
 Token Scanner::scanString(bool const _isUnicode)
 {
-	size_t startPosition = m_source->position();
+	size_t startPosition = m_source.position();
 	char const quote = m_char;
 	advance();  // consume quote
 	LiteralScope literal(this, LITERAL_TYPE_STRING);
@@ -867,7 +852,7 @@ Token Scanner::scanString(bool const _isUnicode)
 
 	if (_isUnicode)
 	{
-		ScannerError unicodeDirectionError = validateBiDiMarkup(*m_source, startPosition);
+		ScannerError unicodeDirectionError = validateBiDiMarkup(m_source, startPosition);
 		if (unicodeDirectionError != ScannerError::NoError)
 			return setError(unicodeDirectionError);
 	}
@@ -921,7 +906,7 @@ void Scanner::scanDecimalDigits()
 	// May continue with decimal digit or underscore for grouping.
 	do
 		addLiteralCharAndAdvance();
-	while (!m_source->isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
+	while (!m_source.isPastEndOfInput() && (isDecimalDigit(m_char) || m_char == '_'));
 
 	// Defer further validation of underscore to SyntaxChecker.
 }
@@ -967,7 +952,7 @@ Token Scanner::scanNumber(char _charSeen)
 			scanDecimalDigits();  // optional
 			if (m_char == '.')
 			{
-				if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
+				if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
 				{
 					// Assume the input may be a floating point number with leading '_' in fraction part.
 					// Recover by consuming it all but returning `Illegal` right away.
@@ -975,7 +960,7 @@ Token Scanner::scanNumber(char _charSeen)
 					addLiteralCharAndAdvance(); // '_'
 					scanDecimalDigits();
 				}
-				if (m_source->isPastEndOfInput() || !isDecimalDigit(m_source->get(1)))
+				if (m_source.isPastEndOfInput() || !isDecimalDigit(m_source.get(1)))
 				{
 					// A '.' has to be followed by a number.
 					literal.complete();
@@ -992,7 +977,7 @@ Token Scanner::scanNumber(char _charSeen)
 		solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number");
 		if (kind != DECIMAL)
 			return setError(ScannerError::IllegalExponent);
-		else if (!m_source->isPastEndOfInput(1) && m_source->get(1) == '_')
+		else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_')
 		{
 			// Recover from wrongly placed underscore as delimiter in literal with scientific
 			// notation by consuming until the end.

liblangutil/Scanner.h

@@ -100,17 +100,13 @@ class Scanner
 {
 	friend class LiteralScope;
 public:
-	explicit Scanner(std::shared_ptr<CharStream> _source) { reset(std::move(_source)); }
+	explicit Scanner(CharStream& _source):
-	explicit Scanner(CharStream _source = CharStream()) { reset(std::move(_source)); }
+		m_source(_source),
+		m_sourceName{std::make_shared<std::string>(_source.name())}
+	{
+		reset();
+	}
 
-	std::string const& source() const noexcept { return m_source->source(); }
-
-	std::shared_ptr<CharStream> charStream() noexcept { return m_source; }
-	std::shared_ptr<CharStream const> charStream() const noexcept { return m_source; }
-
-	/// Resets the scanner as if newly constructed with _source as input.
-	void reset(CharStream _source);
-	void reset(std::shared_ptr<CharStream> _source);
 	/// Resets scanner to the start of input.
 	void reset();
 
@@ -123,6 +119,8 @@ public:
 		rescan();
 	}
 
+	CharStream const& charStream() const noexcept { return m_source; }
+
 	/// @returns the next token and advances input
 	Token next();
 
@@ -201,8 +199,8 @@ private:
 	void addUnicodeAsUTF8(unsigned codepoint);
 	///@}
 
-	bool advance() { m_char = m_source->advanceAndGet(); return !m_source->isPastEndOfInput(); }
+	bool advance() { m_char = m_source.advanceAndGet(); return !m_source.isPastEndOfInput(); }
-	void rollback(size_t _amount) { m_char = m_source->rollback(_amount); }
+	void rollback(size_t _amount) { m_char = m_source.rollback(_amount); }
 	/// Rolls back to the start of the current token and re-runs the scanner.
 	void rescan();
 
@@ -251,15 +249,15 @@ private:
 	bool isUnicodeLinebreak();
 
 	/// Return the current source position.
-	size_t sourcePos() const { return m_source->position(); }
+	size_t sourcePos() const { return m_source.position(); }
-	bool isSourcePastEndOfInput() const { return m_source->isPastEndOfInput(); }
+	bool isSourcePastEndOfInput() const { return m_source.isPastEndOfInput(); }
 
 	enum TokenIndex { Current, Next, NextNext };
 
 	TokenDesc m_skippedComments[3] = {}; // desc for the current, next and nextnext skipped comment
 	TokenDesc m_tokens[3] = {}; // desc for the current, next and nextnext token
 
-	std::shared_ptr<CharStream> m_source;
+	CharStream& m_source;
 	std::shared_ptr<std::string const> m_sourceName;
 
 	ScannerKind m_kind = ScannerKind::Solidity;

libsolidity/ast/ASTJsonImporter.cpp

@@ -958,7 +958,8 @@ Json::Value ASTJsonImporter::member(Json::Value const& _node, string const& _nam
 
 Token ASTJsonImporter::scanSingleToken(Json::Value const& _node)
 {
-	langutil::Scanner scanner{langutil::CharStream(_node.asString(), "")};
+	langutil::CharStream charStream(_node.asString(), "");
+	langutil::Scanner scanner{charStream};
 	astAssert(scanner.peekNextToken() == Token::EOS, "Token string is too long.");
 	return scanner.currentToken();
 }

libsolidity/codegen/CompilerContext.cpp

@@ -434,14 +434,14 @@ void CompilerContext::appendInlineAssembly(
 
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, _sourceName));
+	langutil::CharStream charStream(_assembly, _sourceName);
 	yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
 	optional<langutil::SourceLocation> locationOverride;
 	if (!_system)
 		locationOverride = m_asm->currentSourceLocation();
 	shared_ptr<yul::Block> parserResult =
 		yul::Parser(errorReporter, dialect, std::move(locationOverride))
-		.parse(scanner, false);
+		.parse(charStream);
 #ifdef SOL_OUTPUT_ASM
 	cout << yul::AsmPrinter(&dialect)(*parserResult) << endl;
 #endif
@@ -457,7 +457,7 @@ void CompilerContext::appendInlineAssembly(
 		for (auto const& error: errorReporter.errors())
 			// TODO if we have "locationOverride", it will be the wrong char stream,
 			// but we do not have access to the solidity scanner.
-			message += SourceReferenceFormatter::formatErrorInformation(*error, *scanner->charStream());
+			message += SourceReferenceFormatter::formatErrorInformation(*error, charStream);
 		message += "-------------------------------------------\n";
 
 		solAssert(false, message);
@@ -491,8 +491,8 @@ void CompilerContext::appendInlineAssembly(
 			solAssert(m_generatedYulUtilityCode.empty(), "");
 			m_generatedYulUtilityCode = yul::AsmPrinter(dialect)(*obj.code);
 			string code = yul::AsmPrinter{dialect}(*obj.code);
-			scanner = make_shared<langutil::Scanner>(langutil::CharStream(m_generatedYulUtilityCode, _sourceName));
+			langutil::CharStream charStream(m_generatedYulUtilityCode, _sourceName);
-			obj.code = yul::Parser(errorReporter, dialect).parse(scanner, false);
+			obj.code = yul::Parser(errorReporter, dialect).parse(charStream);
 			*obj.analysisInfo = yul::AsmAnalyzer::analyzeStrictAssertCorrect(dialect, obj);
 		}
 

libsolidity/interface/CompilerStack.cpp

@@ -313,7 +313,7 @@ void CompilerStack::setSources(StringMap _sources)
 	if (m_stackState != Empty)
 		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Must set sources before parsing."));
 	for (auto source: _sources)
-		m_sources[source.first].scanner = make_shared<Scanner>(CharStream(/*content*/std::move(source.second), /*name*/source.first));
+		m_sources[source.first].charStream = make_unique<CharStream>(/*content*/std::move(source.second), /*name*/source.first);
 	m_stackState = SourcesSet;
 }
 
@@ -336,8 +336,7 @@ bool CompilerStack::parse()
 	{
 		string const& path = sourcesToParse[i];
 		Source& source = m_sources[path];
-		source.scanner->reset();
+		source.ast = parser.parse(*source.charStream);
-		source.ast = parser.parse(source.scanner);
 		if (!source.ast)
 			solAssert(!Error::containsOnlyWarnings(m_errorReporter.errors()), "Parser returned null but did not report error.");
 		else
@@ -348,7 +347,7 @@ bool CompilerStack::parse()
 				{
 					string const& newPath = newSource.first;
 					string const& newContents = newSource.second;
-					m_sources[newPath].scanner = make_shared<Scanner>(CharStream(newContents, newPath));
+					m_sources[newPath].charStream = make_shared<CharStream>(newContents, newPath);
 					sourcesToParse.push_back(newPath);
 				}
 		}
@@ -377,10 +376,11 @@ void CompilerStack::importASTs(map<string, Json::Value> const& _sources)
 		string const& path = src.first;
 		Source source;
 		source.ast = src.second;
-		string srcString = util::jsonCompactPrint(m_sourceJsons[src.first]);
+		source.charStream = make_shared<CharStream>(
-		ASTPointer<Scanner> scanner = make_shared<Scanner>(langutil::CharStream(srcString, src.first));
+			util::jsonCompactPrint(m_sourceJsons[src.first]),
-		source.scanner = scanner;
+			src.first
-		m_sources[path] = source;
+		);
+		m_sources[path] = move(source);
 	}
 	m_stackState = ParsedAndImported;
 	m_importedSources = true;
@@ -754,9 +754,9 @@ Json::Value CompilerStack::generatedSources(string const& _contractName, bool _r
 				unsigned sourceIndex = sourceIndices()[sourceName];
 				ErrorList errors;
 				ErrorReporter errorReporter(errors);
-				auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(source, sourceName));
+				CharStream charStream(source, sourceName);
 				yul::EVMDialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(m_evmVersion);
-				shared_ptr<yul::Block> parserResult = yul::Parser{errorReporter, dialect}.parse(scanner, false);
+				shared_ptr<yul::Block> parserResult = yul::Parser{errorReporter, dialect}.parse(charStream);
 				solAssert(parserResult, "");
 				sources[0]["ast"] = yul::AsmJsonConverter{sourceIndex}(*parserResult);
 				sources[0]["name"] = sourceName;
@@ -1031,10 +1031,9 @@ CharStream const& CompilerStack::charStream(string const& _sourceName) const
 	if (m_stackState < SourcesSet)
 		BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("No sources set."));
 
-	solAssert(source(_sourceName).scanner, "");
+	solAssert(source(_sourceName).charStream, "");
-	solAssert(source(_sourceName).scanner->charStream(), "");
 
-	return *source(_sourceName).scanner->charStream();
+	return *source(_sourceName).charStream;
 }
 
 SourceUnit const& CompilerStack::ast(string const& _sourceName) const
@@ -1079,21 +1078,21 @@ size_t CompilerStack::functionEntryPoint(
 h256 const& CompilerStack::Source::keccak256() const
 {
 	if (keccak256HashCached == h256{})
-		keccak256HashCached = util::keccak256(scanner->source());
+		keccak256HashCached = util::keccak256(charStream->source());
 	return keccak256HashCached;
 }
 
 h256 const& CompilerStack::Source::swarmHash() const
 {
 	if (swarmHashCached == h256{})
-		swarmHashCached = util::bzzr1Hash(scanner->source());
+		swarmHashCached = util::bzzr1Hash(charStream->source());
 	return swarmHashCached;
 }
 
 string const& CompilerStack::Source::ipfsUrl() const
 {
 	if (ipfsUrlCached.empty())
-		ipfsUrlCached = "dweb:/ipfs/" + util::ipfsHashBase58(scanner->source());
+		ipfsUrlCached = "dweb:/ipfs/" + util::ipfsHashBase58(charStream->source());
 	return ipfsUrlCached;
 }
 
@@ -1454,12 +1453,12 @@ string CompilerStack::createMetadata(Contract const& _contract) const
 		if (!referencedSources.count(s.first))
 			continue;
 
-		solAssert(s.second.scanner, "Scanner not available");
+		solAssert(s.second.charStream, "Character stream not available");
 		meta["sources"][s.first]["keccak256"] = "0x" + toHex(s.second.keccak256().asBytes());
 		if (optional<string> licenseString = s.second.ast->licenseString())
 			meta["sources"][s.first]["license"] = *licenseString;
 		if (m_metadataLiteralSources)
-			meta["sources"][s.first]["content"] = s.second.scanner->source();
+			meta["sources"][s.first]["content"] = s.second.charStream->source();
 		else
 		{
 			meta["sources"][s.first]["urls"] = Json::arrayValue;

libsolidity/interface/CompilerStack.h

@@ -57,7 +57,6 @@
 
 namespace solidity::langutil
 {
-class Scanner;
 class CharStream;
 }
 
@@ -344,7 +343,7 @@ private:
 	/// The state per source unit. Filled gradually during parsing.
 	struct Source
 	{
-		std::shared_ptr<langutil::Scanner> scanner;
+		std::shared_ptr<langutil::CharStream> charStream;
 		std::shared_ptr<SourceUnit> ast;
 		util::h256 mutable keccak256HashCached;
 		util::h256 mutable swarmHashCached;

libsolidity/parsing/Parser.cpp

@@ -81,13 +81,13 @@ private:
 	SourceLocation m_location;
 };
 
-ASTPointer<SourceUnit> Parser::parse(shared_ptr<Scanner> const& _scanner)
+ASTPointer<SourceUnit> Parser::parse(CharStream& _charStream)
 {
 	solAssert(!m_insideModifier, "");
 	try
 	{
 		m_recursionDepth = 0;
-		m_scanner = _scanner;
+		m_scanner = make_shared<Scanner>(_charStream);
 		ASTNodeFactory nodeFactory(*this);
 
 		vector<ASTPointer<ASTNode>> nodes;
@@ -1300,7 +1300,7 @@ ASTPointer<InlineAssembly> Parser::parseInlineAssembly(ASTPointer<ASTString> con
 	}
 
 	yul::Parser asmParser(m_errorReporter, dialect);
-	shared_ptr<yul::Block> block = asmParser.parse(m_scanner, true);
+	shared_ptr<yul::Block> block = asmParser.parseInline(m_scanner);
 	if (block == nullptr)
 		BOOST_THROW_EXCEPTION(FatalError());
 
@@ -2061,9 +2061,9 @@ optional<string> Parser::findLicenseString(std::vector<ASTPointer<ASTNode>> cons
 
 	// Search inside all parts of the source not covered by parsed nodes.
 	// This will leave e.g. "global comments".
-	string const& source = m_scanner->source();
+	using iter = std::string::const_iterator;
-	using iter = decltype(source.begin());
 	vector<pair<iter, iter>> sequencesToSearch;
+	string const& source = m_scanner->charStream().source();
 	sequencesToSearch.emplace_back(source.begin(), source.end());
 	for (ASTPointer<ASTNode> const& node: _nodes)
 		if (node->location().hasText())

libsolidity/parsing/Parser.h

@@ -29,7 +29,7 @@
 
 namespace solidity::langutil
 {
-class Scanner;
+class CharStream;
 }
 
 namespace solidity::frontend
@@ -47,7 +47,7 @@ public:
 		m_evmVersion(_evmVersion)
 	{}
 
-	ASTPointer<SourceUnit> parse(std::shared_ptr<langutil::Scanner> const& _scanner);
+	ASTPointer<SourceUnit> parse(langutil::CharStream& _charStream);
 
 private:
 	class ASTNodeFactory;

libyul/AsmJsonImporter.cpp

@@ -166,7 +166,8 @@ Literal AsmJsonImporter::createLiteral(Json::Value const& _node)
 
 	if (kind == "number")
 	{
-		langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")};
+		langutil::CharStream charStream(lit.value.str(), "");
+		langutil::Scanner scanner{charStream};
 		lit.kind = LiteralKind::Number;
 		yulAssert(
 			scanner.currentToken() == Token::Number,
@@ -175,7 +176,8 @@ Literal AsmJsonImporter::createLiteral(Json::Value const& _node)
 	}
 	else if (kind == "bool")
 	{
-		langutil::Scanner scanner{langutil::CharStream(lit.value.str(), "")};
+		langutil::CharStream charStream(lit.value.str(), "");
+		langutil::Scanner scanner{charStream};
 		lit.kind = LiteralKind::Boolean;
 		yulAssert(
 			scanner.currentToken() == Token::TrueLiteral ||

libyul/AsmParser.cpp

@@ -85,7 +85,15 @@ std::shared_ptr<DebugData const> Parser::createDebugData() const
 	solAssert(false, "");
 }
 
-unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _reuseScanner)
+unique_ptr<Block> Parser::parse(CharStream& _charStream)
+{
+	m_scanner = make_shared<Scanner>(_charStream);
+	unique_ptr<Block> block = parseInline(m_scanner);
+	expectToken(Token::EOS);
+	return block;
+}
+
+unique_ptr<Block> Parser::parseInline(std::shared_ptr<Scanner> const& _scanner)
 {
 	m_recursionDepth = 0;
 
@@ -97,10 +105,7 @@ unique_ptr<Block> Parser::parse(std::shared_ptr<Scanner> const& _scanner, bool _
 		m_scanner = _scanner;
 		if (m_sourceNames)
 			fetchSourceLocationFromComment();
-		auto block = make_unique<Block>(parseBlock());
+		return make_unique<Block>(parseBlock());
-		if (!_reuseScanner)
-			expectToken(Token::EOS);
-		return block;
 	}
 	catch (FatalError const&)
 	{

libyul/AsmParser.h

@@ -87,9 +87,13 @@ public:
 	{}
 
 	/// Parses an inline assembly block starting with `{` and ending with `}`.
-	/// @param _reuseScanner if true, do check for end of input after the `}`.
 	/// @returns an empty shared pointer on error.
-	std::unique_ptr<Block> parse(std::shared_ptr<langutil::Scanner> const& _scanner, bool _reuseScanner);
+	std::unique_ptr<Block> parseInline(std::shared_ptr<langutil::Scanner> const& _scanner);
+
+	/// Parses an assembly block starting with `{` and ending with `}`
+	/// and expects end of input after the '}'.
+	/// @returns an empty shared pointer on error.
+	std::unique_ptr<Block> parse(langutil::CharStream& _charStream);
 
 protected:
 	langutil::SourceLocation currentLocation() const override

libyul/ObjectParser.cpp

@@ -131,7 +131,8 @@ optional<SourceNameMap> ObjectParser::tryParseSourceNameMapping() const
 
 	solAssert(sm.size() == 2, "");
 	auto text = m_scanner->currentCommentLiteral().substr(static_cast<size_t>(sm.position() + sm.length()));
-	Scanner scanner(make_shared<CharStream>(text, ""));
+	CharStream charStream(text, "");
+	Scanner scanner(charStream);
 	if (scanner.currentToken() == Token::EOS)
 		return SourceNameMap{};
 	SourceNameMap sourceNames;
@@ -168,7 +169,7 @@ optional<SourceNameMap> ObjectParser::tryParseSourceNameMapping() const
 shared_ptr<Block> ObjectParser::parseBlock()
 {
 	Parser parser(m_errorReporter, m_dialect, m_sourceNameMapping);
-	shared_ptr<Block> block = parser.parse(m_scanner, true);
+	shared_ptr<Block> block = parser.parseInline(m_scanner);
 	yulAssert(block || m_errorReporter.hasErrors(), "Invalid block but no error!");
 	return block;
 }

libyul/backends/wasm/EVMToEwasmTranslator.cpp

@@ -125,7 +125,7 @@ void EVMToEwasmTranslator::parsePolyfill()
 {
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	shared_ptr<Scanner> scanner{make_shared<Scanner>(CharStream(
+	CharStream charStream(
 		"{" +
 			string(solidity::yul::wasm::polyfill::Arithmetic) +
 			string(solidity::yul::wasm::polyfill::Bitwise) +
@@ -135,15 +135,15 @@ void EVMToEwasmTranslator::parsePolyfill()
 			string(solidity::yul::wasm::polyfill::Keccak) +
 			string(solidity::yul::wasm::polyfill::Logical) +
 			string(solidity::yul::wasm::polyfill::Memory) +
-		"}", ""))};
+		"}", "");
-	m_polyfill = Parser(errorReporter, WasmDialect::instance()).parse(scanner, false);
+	m_polyfill = Parser(errorReporter, WasmDialect::instance()).parse(charStream);
 	if (!errors.empty())
 	{
 		string message;
 		for (auto const& err: errors)
 			message += langutil::SourceReferenceFormatter::formatErrorInformation(
 				*err,
-				SingletonCharStreamProvider(*scanner->charStream())
+				SingletonCharStreamProvider(charStream)
 			);
 		yulAssert(false, message);
 	}

test/liblangutil/Scanner.cpp

@@ -34,13 +34,15 @@ BOOST_AUTO_TEST_SUITE(ScannerTest)
 
 BOOST_AUTO_TEST_CASE(test_empty)
 {
-	Scanner scanner(CharStream{});
+	CharStream stream{};
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(smoke_test)
 {
-	Scanner scanner(CharStream("function break;765  \t  \"string1\",'string2'\nidentifier1", ""));
+	CharStream stream("function break;765  \t  \"string1\",'string2'\nidentifier1", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Break);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
@@ -58,7 +60,8 @@ BOOST_AUTO_TEST_CASE(smoke_test)
 
 BOOST_AUTO_TEST_CASE(assembly_assign)
 {
-	Scanner scanner(CharStream("let a := 1", ""));
+	CharStream stream("let a := 1", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Let);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
@@ -69,7 +72,8 @@ BOOST_AUTO_TEST_CASE(assembly_assign)
 
 BOOST_AUTO_TEST_CASE(assembly_multiple_assign)
 {
-	Scanner scanner(CharStream("let a, b, c := 1", ""));
+	CharStream stream("let a, b, c := 1", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Let);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
@@ -89,14 +93,16 @@ BOOST_AUTO_TEST_CASE(string_printable)
 		// Escape \ and " (since we are quoting with ")
 		if (v == '\\' || v == '"')
 			lit = string{'\\'} + lit;
-		Scanner scanner(CharStream("  { \"" + lit + "\"", ""));
+		CharStream stream("  { \"" + lit + "\"", "");
+		Scanner scanner(stream);
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), string{static_cast<char>(v)});
 		BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 	}
 	// Special case of unescaped " for strings quoted with '
-	Scanner scanner(CharStream("  { '\"'", ""));
+	CharStream stream("  { '\"'", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "\"");
@@ -110,7 +116,8 @@ BOOST_AUTO_TEST_CASE(string_nonprintable)
 		if (v >= 0x20 && v <= 0x7e)
 			continue;
 		string lit{static_cast<char>(v)};
-		Scanner scanner(CharStream("  { \"" + lit + "\"", ""));
+		CharStream stream("  { \"" + lit + "\"", "");
+		Scanner scanner(stream);
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 		if (v == '\n' || v == '\v' || v == '\f' || v == '\r')
@@ -123,7 +130,8 @@ BOOST_AUTO_TEST_CASE(string_nonprintable)
 
 BOOST_AUTO_TEST_CASE(string_escapes)
 {
-	Scanner scanner(CharStream("  { \"a\\x61\"", ""));
+	CharStream stream("  { \"a\\x61\"", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa");
@@ -131,25 +139,46 @@ BOOST_AUTO_TEST_CASE(string_escapes)
 
 BOOST_AUTO_TEST_CASE(string_escapes_all)
 {
-	Scanner scanner(CharStream("  { \"a\\x61\\n\\r\\t\"", ""));
+	CharStream stream("  { \"a\\x61\\n\\r\\t\"", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa\n\r\t");
 }
 
+struct TestScanner
+{
+	unique_ptr<CharStream> stream;
+	unique_ptr<Scanner> scanner;
+	explicit TestScanner(string _text) { reset(move(_text)); }
+
+	void reset(std::string _text)
+	{
+		stream = make_unique<CharStream>(move(_text), "");
+		scanner = make_unique<Scanner>(*stream);
+	}
+
+	decltype(auto) currentToken() { return scanner->currentToken(); }
+	decltype(auto) next() { return scanner->next(); }
+	decltype(auto) currentError() { return scanner->currentError(); }
+	decltype(auto) currentLiteral() { return scanner->currentLiteral(); }
+	decltype(auto) currentCommentLiteral() { return scanner->currentCommentLiteral(); }
+	decltype(auto) currentLocation() { return scanner->currentLocation(); }
+};
+
 BOOST_AUTO_TEST_CASE(string_escapes_legal_before_080)
 {
-	Scanner scanner(CharStream("  { \"a\\b", ""));
+	TestScanner scanner("  { \"a\\b");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
-	scanner.reset(CharStream("  { \"a\\f", ""));
+	scanner.reset("  { \"a\\f");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "");
-	scanner.reset(CharStream("  { \"a\\v", ""));
+	scanner.reset("  { \"a\\v");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@@ -158,7 +187,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_legal_before_080)
 
 BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
 {
-	Scanner scanner(CharStream("  { \"a\\x61\\x00abc\"", ""));
+	TestScanner scanner("  { \"a\\x61\\x00abc\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("aa\0abc", 6));
@@ -166,7 +195,8 @@ BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
 
 BOOST_AUTO_TEST_CASE(string_escape_illegal)
 {
-	Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)", ""));
+	CharStream stream(" bla \"\\x6rf\" (illegalescape)", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@@ -180,7 +210,7 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal)
 
 BOOST_AUTO_TEST_CASE(hex_numbers)
 {
-	Scanner scanner(CharStream("var x = 0x765432536763762734623472346;", ""));
+	TestScanner scanner("var x = 0x765432536763762734623472346;");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -188,34 +218,35 @@ BOOST_AUTO_TEST_CASE(hex_numbers)
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x765432536763762734623472346");
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("0x1234", ""));
+	scanner.reset("0x1234");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234");
-	scanner.reset(CharStream("0X1234", ""));
+	scanner.reset("0X1234");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 }
 
 BOOST_AUTO_TEST_CASE(octal_numbers)
 {
-	Scanner scanner(CharStream("07", ""));
+	TestScanner scanner("07");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
-	scanner.reset(CharStream("007", ""));
+	scanner.reset("007");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
-	scanner.reset(CharStream("-07", ""));
+	scanner.reset("-07");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-	scanner.reset(CharStream("-.07", ""));
+	scanner.reset("-.07");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
-	scanner.reset(CharStream("0", ""));
+	scanner.reset("0");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
-	scanner.reset(CharStream("0.1", ""));
+	scanner.reset("0.1");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 }
 
 BOOST_AUTO_TEST_CASE(scientific_notation)
 {
-	Scanner scanner(CharStream("var x = 2e10;", ""));
+	CharStream stream("var x = 2e10;", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -227,14 +258,14 @@ BOOST_AUTO_TEST_CASE(scientific_notation)
 
 BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
 {
-	Scanner scanner(CharStream("function .a(", ""));
+	TestScanner scanner("function .a(");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("function .a(", ""));
+	scanner.reset("function .a(");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -244,7 +275,7 @@ BOOST_AUTO_TEST_CASE(leading_dot_in_identifier)
 
 BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
 {
-	Scanner scanner(CharStream("function a..a(", ""));
+	TestScanner scanner("function a..a(");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
@@ -252,8 +283,8 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("function a...a(", ""));
+	scanner.reset("function a...a(");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -262,14 +293,14 @@ BOOST_AUTO_TEST_CASE(middle_dot_in_identifier)
 
 BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
 {
-	Scanner scanner(CharStream("function a.(", ""));
+	TestScanner scanner("function a.(");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("function a.(", ""));
+	scanner.reset("function a.(");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -278,19 +309,19 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_identifier)
 
 BOOST_AUTO_TEST_CASE(trailing_dot_in_numbers)
 {
-	Scanner scanner(CharStream("2.5", ""));
+	TestScanner scanner("2.5");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("2.5e10", ""));
+	scanner.reset("2.5e10");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(".5", ""));
+	scanner.reset(".5");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(".5e10", ""));
+	scanner.reset(".5e10");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("2.", ""));
+	scanner.reset("2.");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -299,7 +330,8 @@ BOOST_AUTO_TEST_CASE(trailing_dot_in_numbers)
 BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("_1.2", ""));
+	CharStream stream("_1.2", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -308,11 +340,11 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
 BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("1._2", ""));
+	TestScanner scanner("1._2");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-	scanner.reset(CharStream("1._", ""));
+	scanner.reset("1._");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -320,7 +352,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
 BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("_1e2", ""));
+	CharStream stream("_1e2", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -328,7 +361,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
 BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("1e_2", ""));
+	CharStream stream("1e_2", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2");
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -336,7 +370,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
 
 BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
 {
-	Scanner scanner(CharStream("0x_abc", ""));
+	CharStream stream("0x_abc", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -345,7 +380,8 @@ BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
 BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("12._1234_1234", ""));
+	CharStream stream("12._1234_1234", "");
+	Scanner scanner(stream);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -353,22 +389,22 @@ BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
 BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("0x123_", ""));
+	TestScanner scanner("0x123_");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-	scanner.reset(CharStream("123_", ""));
+	scanner.reset("123_");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-	scanner.reset(CharStream("12.34_", ""));
+	scanner.reset("12.34_");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(negative_numbers)
 {
-	Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;", ""));
+	TestScanner scanner("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -395,7 +431,7 @@ BOOST_AUTO_TEST_CASE(negative_numbers)
 
 BOOST_AUTO_TEST_CASE(locations)
 {
-	Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment", ""));
+	TestScanner scanner("function_identifier has ; -0x743/*comment*/\n ident //comment");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.currentLocation().start, 0);
 	BOOST_CHECK_EQUAL(scanner.currentLocation().end, 19);
@@ -418,7 +454,7 @@ BOOST_AUTO_TEST_CASE(locations)
 BOOST_AUTO_TEST_CASE(ambiguities)
 {
 	// test scanning of some operators which need look-ahead
-	Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=", ""));
+	TestScanner scanner("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LessThanOrEqual);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LessThan);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Add);
@@ -440,21 +476,21 @@ BOOST_AUTO_TEST_CASE(ambiguities)
 
 BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin)
 {
-	Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user", ""));
+	TestScanner scanner("/// Send $(value / 1000) chocolates to the user");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
 }
 
 BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed_begin)
 {
-	Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/", ""));
+	TestScanner scanner("/** Send $(value / 1000) chocolates to the user*/");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
 }
 
 BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
 {
-	Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user", ""));
+	TestScanner scanner("some other tokens /// Send $(value / 1000) chocolates to the user");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -464,9 +500,9 @@ BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
 
 BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
 {
-	Scanner scanner(CharStream("some other tokens /**\n"
+	TestScanner scanner("some other tokens /**\n"
 						"* Send $(value / 1000) chocolates to the user\n"
-							   "*/", ""));
+						"*/");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -476,9 +512,9 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
 
 BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
 {
-	Scanner scanner(CharStream("some other tokens /**\n"
+	TestScanner scanner("some other tokens /**\n"
 						" Send $(value / 1000) chocolates to the user\n"
-							   "*/", ""));
+						"*/");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -488,9 +524,9 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
 
 BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
 {
-	Scanner scanner(CharStream("some other tokens /** \t \r \n"
+	TestScanner scanner("some other tokens /** \t \r \n"
 						"\t \r  * Send $(value / 1000) chocolates to the user\n"
-							   "*/", ""));
+						"*/");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -500,37 +536,37 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
 
 BOOST_AUTO_TEST_CASE(comment_before_eos)
 {
-	Scanner scanner(CharStream("//", ""));
+	TestScanner scanner("//");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(documentation_comment_before_eos)
 {
-	Scanner scanner(CharStream("///", ""));
+	TestScanner scanner("///");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(empty_multiline_comment)
 {
-	Scanner scanner(CharStream("/**/", ""));
+	TestScanner scanner("/**/");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(empty_multiline_documentation_comment_before_eos)
 {
-	Scanner scanner(CharStream("/***/", ""));
+	TestScanner scanner("/***/");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
 {
-	Scanner scanner(CharStream("hello_world ///documentation comment \n"
+	TestScanner scanner("hello_world ///documentation comment \n"
 						"//simple comment \n"
-							   "<<", ""));
+						"<<");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SHL);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "documentation comment ");
@@ -538,7 +574,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
 
 BOOST_AUTO_TEST_CASE(ether_subdenominations)
 {
-	Scanner scanner(CharStream("wei gwei ether", ""));
+	TestScanner scanner("wei gwei ether");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubWei);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubGwei);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubEther);
@@ -546,7 +582,7 @@ BOOST_AUTO_TEST_CASE(ether_subdenominations)
 
 BOOST_AUTO_TEST_CASE(time_subdenominations)
 {
-	Scanner scanner(CharStream("seconds minutes hours days weeks years", ""));
+	TestScanner scanner("seconds minutes hours days weeks years");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubSecond);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubMinute);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubHour);
@@ -557,7 +593,7 @@ BOOST_AUTO_TEST_CASE(time_subdenominations)
 
 BOOST_AUTO_TEST_CASE(empty_comment)
 {
-	Scanner scanner(CharStream("//\ncontract{}", ""));
+	TestScanner scanner("//\ncontract{}");
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Contract);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LBrace);
@@ -569,7 +605,7 @@ BOOST_AUTO_TEST_CASE(empty_comment)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
 {
-	Scanner scanner(CharStream("{ \"\\u00DAnicode\"", ""));
+	TestScanner scanner("{ \"\\u00DAnicode\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xC3\x9Anicode", 8));
@@ -577,7 +613,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
 {
-	Scanner scanner(CharStream("{ \"\\u007Fnicode\"", ""));
+	TestScanner scanner("{ \"\\u007Fnicode\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x7Fnicode", 7));
@@ -585,7 +621,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
 {
-	Scanner scanner(CharStream("{ \"\\u07FFnicode\"", ""));
+	TestScanner scanner("{ \"\\u07FFnicode\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xDF\xBFnicode", 8));
@@ -593,7 +629,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
 {
-	Scanner scanner(CharStream("{ \"\\uFFFFnicode\"", ""));
+	TestScanner scanner("{ \"\\uFFFFnicode\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xEF\xBF\xBFnicode", 9));
@@ -601,7 +637,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
 
 BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
 {
-	Scanner scanner(CharStream("{ \"\\uFFnicode\"", ""));
+	TestScanner scanner("{ \"\\uFFnicode\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 }
@@ -610,12 +646,12 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
 
 BOOST_AUTO_TEST_CASE(unicode_prefix_only)
 {
-	Scanner scanner(CharStream("{ unicode", ""));
+	TestScanner scanner("{ unicode");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
-	scanner.reset(CharStream("{ unicode", ""));
+	scanner.reset("{ unicode");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "unicode");
@@ -623,7 +659,7 @@ BOOST_AUTO_TEST_CASE(unicode_prefix_only)
 
 BOOST_AUTO_TEST_CASE(unicode_invalid_space)
 {
-	Scanner scanner(CharStream("{ unicode ", ""));
+	TestScanner scanner("{ unicode ");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@@ -631,12 +667,12 @@ BOOST_AUTO_TEST_CASE(unicode_invalid_space)
 
 BOOST_AUTO_TEST_CASE(unicode_invalid_token)
 {
-	Scanner scanner(CharStream("{ unicode test", ""));
+	TestScanner scanner("{ unicode test");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
-	scanner.reset(CharStream("{ unicode test", ""));
+	scanner.reset("{ unicode test");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "unicode");
@@ -646,7 +682,7 @@ BOOST_AUTO_TEST_CASE(unicode_invalid_token)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_literal)
 {
-	Scanner scanner(CharStream("{ unicode\"Hello 😃\"", ""));
+	TestScanner scanner("{ unicode\"Hello 😃\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::UnicodeStringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("Hello \xf0\x9f\x98\x83", 10));
@@ -655,7 +691,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_literal)
 BOOST_AUTO_TEST_CASE(valid_nonprintable_in_unicode_literal)
 {
 	// Non-printable characters are allowed in unicode strings...
-	Scanner scanner(CharStream("{ unicode\"Hello \007😃\"", ""));
+	TestScanner scanner("{ unicode\"Hello \007😃\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::UnicodeStringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("Hello \x07\xf0\x9f\x98\x83", 11));
@@ -665,19 +701,19 @@ BOOST_AUTO_TEST_CASE(valid_nonprintable_in_unicode_literal)
 
 BOOST_AUTO_TEST_CASE(hex_prefix_only)
 {
-	Scanner scanner(CharStream("{ hex", ""));
+	TestScanner scanner("{ hex");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
-	scanner.reset(CharStream("{ hex", ""));
+	scanner.reset("{ hex");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
 }
 
 BOOST_AUTO_TEST_CASE(hex_invalid_space)
 {
-	Scanner scanner(CharStream("{ hex ", ""));
+	TestScanner scanner("{ hex ");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@@ -685,12 +721,12 @@ BOOST_AUTO_TEST_CASE(hex_invalid_space)
 
 BOOST_AUTO_TEST_CASE(hex_invalid_token)
 {
-	Scanner scanner(CharStream("{ hex test", ""));
+	TestScanner scanner("{ hex test");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
-	scanner.reset(CharStream("{ hex test", ""));
+	scanner.reset("{ hex test");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalToken);
@@ -698,7 +734,7 @@ BOOST_AUTO_TEST_CASE(hex_invalid_token)
 
 BOOST_AUTO_TEST_CASE(valid_hex_literal)
 {
-	Scanner scanner(CharStream("{ hex\"00112233FF\"", ""));
+	TestScanner scanner("{ hex\"00112233FF\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::HexStringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x00\x11\x22\x33\xFF", 5));
@@ -706,7 +742,7 @@ BOOST_AUTO_TEST_CASE(valid_hex_literal)
 
 BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
 {
-	Scanner scanner(CharStream("{ hex\"00112233F\"", ""));
+	TestScanner scanner("{ hex\"00112233F\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -714,7 +750,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
 
 BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
 {
-	Scanner scanner(CharStream("{ hex\"00112233FF \"", ""));
+	TestScanner scanner("{ hex\"00112233FF \"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -722,7 +758,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
 
 BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
 {
-	Scanner scanner(CharStream("{ hex\"00112233FF'", ""));
+	TestScanner scanner("{ hex\"00112233FF'");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -730,7 +766,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
 
 BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
 {
-	Scanner scanner(CharStream("{ hex\"hello\"", ""));
+	TestScanner scanner("{ hex\"hello\"");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -741,7 +777,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
 BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
 {
 	// This used to parse as "comment", "identifier"
-	Scanner scanner(CharStream("/** / x", ""));
+	TestScanner scanner("/** / x");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -749,14 +785,14 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
 BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos)
 {
 	// This used to parse as "whitespace"
-	Scanner scanner(CharStream("/**", ""));
+	TestScanner scanner("/**");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(multiline_comment_at_eos)
 {
-	Scanner scanner(CharStream("/*", ""));
+	TestScanner scanner("/*");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -765,7 +801,7 @@ BOOST_AUTO_TEST_CASE(regular_line_break_in_single_line_comment)
 {
 	for (auto const& nl: {"\r", "\n", "\r\n"})
 	{
-		Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
+		TestScanner scanner("// abc " + string(nl) + " def ");
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -777,7 +813,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_comment)
 {
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
-		Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
+		TestScanner scanner("// abc " + string(nl) + " def ");
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		for (size_t i = 0; i < string(nl).size() - 1; i++)
@@ -792,7 +828,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_single_line_doc_comment)
 {
 	for (auto const& nl: {"\r", "\n", "\r\n"})
 	{
-		Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
+		TestScanner scanner("/// abc " + string(nl) + " def ");
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -806,7 +842,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_multiline_doc_comment)
 	// Any accepted non-LF is being canonicalized to LF.
 	for (auto const& nl : {"\r"s, "\n"s, "\r\n"s})
 	{
-		Scanner scanner{CharStream{"/// Hello" + nl + "/// World" + nl + "ident", ""}};
+		TestScanner scanner{"/// Hello" + nl + "/// World" + nl + "ident"};
 		auto const& lit = scanner.currentCommentLiteral();
 		BOOST_CHECK_EQUAL(lit, "Hello\n World");
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Hello\n World");
@@ -820,7 +856,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_doc_comment)
 {
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
-		Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
+		TestScanner scanner("/// abc " + string(nl) + " def ");
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		for (size_t i = 0; i < string(nl).size() - 1; i++)
@@ -835,7 +871,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings)
 {
 	for (auto const& nl: {"\r"s, "\n"s, "\r\n"s})
 	{
-		Scanner scanner(CharStream("\"abc " + nl + " def\"", ""));
+		TestScanner scanner("\"abc " + nl + " def\"");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -848,7 +884,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings)
 {
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
-		Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
+		TestScanner scanner("\"abc " + string(nl) + " def\"");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		for (size_t i = 0; i < string(nl).size(); i++)
 			BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
@@ -863,7 +899,7 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
 {
 	// These are tokens which have a different meaning in Yul.
 	string keywords = "return byte bool address var in true false leave switch case default";
-	Scanner scanner(CharStream(keywords, ""));
+	TestScanner scanner(keywords);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Return);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Byte);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Bool);
@@ -877,8 +913,8 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Case);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Default);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(keywords, ""));
+	scanner.reset(keywords);
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -896,28 +932,28 @@ BOOST_AUTO_TEST_CASE(solidity_keywords)
 
 BOOST_AUTO_TEST_CASE(yul_keyword_like)
 {
-	Scanner scanner(CharStream("leave.function", ""));
+	TestScanner scanner("leave.function");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("leave.function", ""));
+	scanner.reset("leave.function");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(yul_identifier_with_dots)
 {
-	Scanner scanner(CharStream("mystorage.slot := 1", ""));
+	TestScanner scanner("mystorage.slot := 1");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("mystorage.slot := 1", ""));
+	scanner.reset("mystorage.slot := 1");
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::AssemblyAssign);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
@@ -927,7 +963,7 @@ BOOST_AUTO_TEST_CASE(yul_identifier_with_dots)
 BOOST_AUTO_TEST_CASE(yul_function)
 {
 	string sig = "function f(a, b) -> x, y";
-	Scanner scanner(CharStream(sig, ""));
+	TestScanner scanner(sig);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -940,8 +976,8 @@ BOOST_AUTO_TEST_CASE(yul_function)
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(sig, ""));
+	scanner.reset(sig);
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -959,7 +995,7 @@ BOOST_AUTO_TEST_CASE(yul_function)
 BOOST_AUTO_TEST_CASE(yul_function_with_whitespace)
 {
 	string sig = "function f (a, b) - > x, y";
-	Scanner scanner(CharStream(sig, ""));
+	TestScanner scanner(sig);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);
@@ -973,8 +1009,8 @@ BOOST_AUTO_TEST_CASE(yul_function_with_whitespace)
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Comma);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(sig, ""));
+	scanner.reset(sig);
-	scanner.setScannerMode(ScannerKind::Yul);
+	scanner.scanner->setScannerMode(ScannerKind::Yul);
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LParen);

test/libsolidity/Assembly.cpp

@@ -26,7 +26,8 @@
 #include <liblangutil/SourceLocation.h>
 #include <libevmasm/Assembly.h>
 
-#include <liblangutil/Scanner.h>
+#include <liblangutil/CharStream.h>
+
 #include <libsolidity/parsing/Parser.h>
 #include <libsolidity/analysis/DeclarationTypeChecker.h>
 #include <libsolidity/analysis/NameAndTypeResolver.h>
@@ -58,7 +59,7 @@ evmasm::AssemblyItems compileContract(std::shared_ptr<CharStream> _sourceCode)
 	ErrorReporter errorReporter(errors);
 	Parser parser(errorReporter, solidity::test::CommonOptions::get().evmVersion());
 	ASTPointer<SourceUnit> sourceUnit;
-	BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(_sourceCode)));
+	BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(*_sourceCode));
 	BOOST_CHECK(!!sourceUnit);
 
 	Scoper::assignScopes(*sourceUnit);

test/libsolidity/SemVerMatcher.cpp

@@ -43,7 +43,8 @@ namespace
 
 SemVerMatchExpression parseExpression(string const& _input)
 {
-	Scanner scanner{CharStream(_input, "")};
+	CharStream stream(_input, "");
+	Scanner scanner{stream};
 	vector<string> literals;
 	vector<Token> tokens;
 	while (scanner.currentToken() != Token::EOS)

test/libsolidity/SolidityExpressionCompiler.cpp

@@ -101,15 +101,14 @@ bytes compileFirstExpression(
 )
 {
 	string sourceCode = "pragma solidity >=0.0; // SPDX-License-Identifier: GPL-3\n" + _sourceCode;
+	CharStream stream(sourceCode, "");
 
 	ASTPointer<SourceUnit> sourceUnit;
 	try
 	{
 		ErrorList errors;
 		ErrorReporter errorReporter(errors);
-		sourceUnit = Parser(errorReporter, solidity::test::CommonOptions::get().evmVersion()).parse(
+		sourceUnit = Parser(errorReporter, solidity::test::CommonOptions::get().evmVersion()).parse(stream);
-			make_shared<Scanner>(CharStream(sourceCode, ""))
-		);
 		if (!sourceUnit)
 			return bytes();
 	}

test/libsolidity/SolidityParser.cpp

@@ -42,11 +42,12 @@ namespace
 ASTPointer<ContractDefinition> parseText(std::string const& _source, ErrorList& _errors, bool errorRecovery = false)
 {
 	ErrorReporter errorReporter(_errors);
+	auto charStream = CharStream(_source, "");
 	ASTPointer<SourceUnit> sourceUnit = Parser(
 		errorReporter,
 		solidity::test::CommonOptions::get().evmVersion(),
 		errorRecovery
-	).parse(std::make_shared<Scanner>(CharStream(_source, "")));
+	).parse(charStream);
 	if (!sourceUnit)
 		return ASTPointer<ContractDefinition>();
 	for (ASTPointer<ASTNode> const& node: sourceUnit->nodes())

test/libyul/Common.cpp

@@ -74,7 +74,8 @@ pair<shared_ptr<Object>, shared_ptr<yul::AsmAnalysisInfo>> yul::test::parse(
 )
 {
 	ErrorReporter errorReporter(_errors);
-	shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_source, ""));
+	CharStream stream(_source, "");
+	shared_ptr<Scanner> scanner = make_shared<Scanner>(stream);
 	shared_ptr<Object> parserResult = yul::ObjectParser(errorReporter, _dialect).parse(scanner, false);
 	if (!parserResult)
 		return {};

test/libyul/ObjectParser.cpp

@@ -23,6 +23,8 @@
 
 #include <test/libsolidity/ErrorCheck.h>
 
+#include <liblangutil/Scanner.h>
+
 #include <libyul/AssemblyStack.h>
 #include <libyul/backends/evm/EVMDialect.h>
 
@@ -118,7 +120,8 @@ tuple<optional<SourceNameMap>, ErrorList> tryGetSourceLocationMapping(string _so
 	ErrorReporter reporter(errors);
 	Dialect const& dialect = yul::EVMDialect::strictAssemblyForEVM(EVMVersion::berlin());
 	ObjectParser objectParser{reporter, dialect};
-	objectParser.parse(make_shared<Scanner>(CharStream(move(source), "")), false);
+	CharStream stream(move(source), "");
+	objectParser.parse(make_shared<Scanner>(stream), false);
 	return {objectParser.sourceNameMapping(), std::move(errors)};
 }
 

test/libyul/Parser.cpp

@@ -30,7 +30,6 @@
 #include <libyul/AsmAnalysis.h>
 #include <libyul/AsmAnalysisInfo.h>
 #include <libyul/Dialect.h>
-#include <liblangutil/Scanner.h>
 #include <liblangutil/ErrorReporter.h>
 
 #include <boost/algorithm/string/replace.hpp>
@@ -55,7 +54,7 @@ shared_ptr<Block> parse(string const& _source, Dialect const& _dialect, ErrorRep
 {
 	try
 	{
-		auto scanner = make_shared<Scanner>(CharStream(_source, ""));
+		auto stream = CharStream(_source, "");
 		map<unsigned, shared_ptr<string const>> indicesToSourceNames;
 		indicesToSourceNames[0] = make_shared<string const>("source0");
 		indicesToSourceNames[1] = make_shared<string const>("source1");
@@ -64,7 +63,7 @@ shared_ptr<Block> parse(string const& _source, Dialect const& _dialect, ErrorRep
 			errorReporter,
 			_dialect,
 			move(indicesToSourceNames)
-		).parse(scanner, false);
+		).parse(stream);
 		if (parserResult)
 		{
 			yul::AsmAnalysisInfo analysisInfo;

test/tools/yulopti.cpp

@@ -22,7 +22,6 @@
 #include <libsolutil/CommonIO.h>
 #include <libsolutil/Exceptions.h>
 #include <liblangutil/ErrorReporter.h>
-#include <liblangutil/Scanner.h>
 #include <libyul/AsmAnalysis.h>
 #include <libyul/AsmAnalysisInfo.h>
 #include <libsolidity/parsing/Parser.h>
@@ -45,7 +44,6 @@
 
 #include <libsolidity/interface/OptimiserSettings.h>
 #include <liblangutil/CharStreamProvider.h>
-#include <liblangutil/Scanner.h>
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/join.hpp>
@@ -82,7 +80,7 @@ public:
 	{
 		SourceReferenceFormatter{
 			cerr,
-			SingletonCharStreamProvider(*m_scanner->charStream()),
+			SingletonCharStreamProvider(*m_charStream),
 			true,
 			false
 		}.printErrorInformation(m_errors);
@@ -91,8 +89,8 @@ public:
 	bool parse(string const& _input)
 	{
 		ErrorReporter errorReporter(m_errors);
-		m_scanner = make_shared<Scanner>(CharStream(_input, ""));
+		m_charStream = make_shared<CharStream>(_input, "");
-		m_ast = yul::Parser(errorReporter, m_dialect).parse(m_scanner, false);
+		m_ast = yul::Parser(errorReporter, m_dialect).parse(*m_charStream);
 		if (!m_ast || !errorReporter.errors().empty())
 		{
 			cerr << "Error parsing source." << endl;
@@ -234,7 +232,7 @@ public:
 
 private:
 	ErrorList m_errors;
-	shared_ptr<Scanner> m_scanner;
+	shared_ptr<CharStream> m_charStream;
 	shared_ptr<yul::Block> m_ast;
 	Dialect const& m_dialect{EVMDialect::strictAssemblyForEVMObjects(EVMVersion{})};
 	shared_ptr<AsmAnalysisInfo> m_analysisInfo;

tools/yulPhaser/Program.cpp

@@ -120,7 +120,7 @@ variant<unique_ptr<Block>, ErrorList> Program::parseObject(Dialect const& _diale
 {
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	auto scanner = make_shared<Scanner>(move(_source));
+	auto scanner = make_shared<Scanner>(_source);
 
 	ObjectParser parser(errorReporter, _dialect);
 	shared_ptr<Object> object = parser.parse(scanner, false);