Merge pull request #5860 from ethereum/test-file-parser

Test file parser.

Changelog.md

@@ -23,6 +23,7 @@ Build System:
  * Add support for continuous fuzzing via Google oss-fuzz
  * SMT: If using Z3, require version 4.6.0 or newer.
  * Ubuntu PPA Packages: Use CVC4 as SMT solver instead of Z3
+ * Soltest: Add parser that is used in the file-based unit test environment.
 
 
 ### 0.5.3 (2019-01-22)

test/CMakeLists.txt

@@ -17,6 +17,8 @@ if (LLL)
 endif()
 file(GLOB libsolidity_sources "libsolidity/*.cpp")
 file(GLOB libsolidity_headers "libsolidity/*.h")
+file(GLOB libsolidity_util_sources "libsolidity/util/*.cpp")
+file(GLOB libsolidity_util_headers "libsolidity/util/*.h")
 
 add_executable(soltest ${sources} ${headers}
     ${contracts_sources} ${contracts_headers}
@@ -26,6 +28,7 @@ add_executable(soltest ${sources} ${headers}
     ${libyul_sources} ${libyul_headers}
     ${liblll_sources} ${liblll_headers}
     ${libsolidity_sources} ${libsolidity_headers}
+    ${libsolidity_util_sources} ${libsolidity_util_headers}
 )
 target_link_libraries(soltest PRIVATE libsolc yul solidity evmasm devcore ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
 

test/libsolidity/util/TestFileParser.cpp

@@ -0,0 +1,401 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/util/TestFileParser.h>
+
+#include <test/Options.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/optional.hpp>
+#include <boost/throw_exception.hpp>
+#include <fstream>
+#include <memory>
+#include <stdexcept>
+
+using namespace dev;
+using namespace langutil;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+using namespace soltest;
+
+namespace
+{
+	bool isIdentifierStart(char c)
+	{
+		return c == '_' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
+	}
+	bool isIdentifierPart(char c)
+	{
+		return isIdentifierStart(c) || isdigit(c);
+	}
+}
+
+vector<dev::solidity::test::FunctionCall> TestFileParser::parseFunctionCalls()
+{
+	vector<FunctionCall> calls;
+	if (!accept(Token::EOS))
+	{
+		assert(m_scanner.currentToken() == Token::Unknown);
+		m_scanner.scanNextToken();
+
+		while (!accept(Token::EOS))
+		{
+			if (!accept(Token::Whitespace))
+			{
+				FunctionCall call;
+
+				/// If this is not the first call in the test,
+				/// the last call to parseParameter could have eaten the
+				/// new line already. This could only be fixed with a one
+				/// token lookahead that checks parseParameter
+				/// if the next token is an identifier.
+				if (calls.empty())
+					expect(Token::Newline);
+				else
+					accept(Token::Newline, true);
+
+				call.signature = parseFunctionSignature();
+				if (accept(Token::Comma, true))
+					call.value = parseFunctionCallValue();
+				if (accept(Token::Colon, true))
+					call.arguments = parseFunctionCallArguments();
+
+				if (accept(Token::Newline, true))
+					call.displayMode = FunctionCall::DisplayMode::MultiLine;
+
+				call.arguments.comment = parseComment();
+
+				if (accept(Token::Newline, true))
+					call.displayMode = FunctionCall::DisplayMode::MultiLine;
+
+				expect(Token::Arrow);
+				call.expectations = parseFunctionCallExpectations();
+				call.expectations.comment = parseComment();
+
+				calls.emplace_back(std::move(call));
+			}
+		}
+	}
+	return calls;
+}
+
+bool TestFileParser::accept(soltest::Token _token, bool const _expect)
+{
+	if (m_scanner.currentToken() != _token)
+		return false;
+	if (_expect)
+		return expect(_token);
+	return true;
+}
+
+bool TestFileParser::expect(soltest::Token _token, bool const _advance)
+{
+	if (m_scanner.currentToken() != _token || m_scanner.currentToken() == Token::Invalid)
+		throw Error(
+			Error::Type::ParserError,
+			"Unexpected " + formatToken(m_scanner.currentToken()) + ": \"" +
+			m_scanner.currentLiteral() + "\". " +
+			"Expected \"" + formatToken(_token) + "\"."
+			);
+	if (_advance)
+		m_scanner.scanNextToken();
+	return true;
+}
+
+string TestFileParser::parseFunctionSignature()
+{
+	string signature = m_scanner.currentLiteral();
+	expect(Token::Identifier);
+
+	signature += formatToken(Token::LParen);
+	expect(Token::LParen);
+
+	string parameters;
+	if (!accept(Token::RParen, false))
+		parameters = parseIdentifierOrTuple();
+
+	while (accept(Token::Comma))
+	{
+		parameters += formatToken(Token::Comma);
+		expect(Token::Comma);
+		parameters += parseIdentifierOrTuple();
+	}
+	if (accept(Token::Arrow, true))
+		throw Error(Error::Type::ParserError, "Invalid signature detected: " + signature);
+
+	signature += parameters;
+
+	expect(Token::RParen);
+	signature += formatToken(Token::RParen);
+	return signature;
+}
+
+u256 TestFileParser::parseFunctionCallValue()
+{
+	u256 value = convertNumber(parseNumber());
+	expect(Token::Ether);
+	return value;
+}
+
+FunctionCallArgs TestFileParser::parseFunctionCallArguments()
+{
+	FunctionCallArgs arguments;
+
+	auto param = parseParameter();
+	if (param.abiType.type == ABIType::None)
+		throw Error(Error::Type::ParserError, "No argument provided.");
+	arguments.parameters.emplace_back(param);
+
+	while (accept(Token::Comma, true))
+		arguments.parameters.emplace_back(parseParameter());
+	return arguments;
+}
+
+FunctionCallExpectations TestFileParser::parseFunctionCallExpectations()
+{
+	FunctionCallExpectations expectations;
+
+	auto param = parseParameter();
+	if (param.abiType.type == ABIType::None)
+	{
+		expectations.failure = false;
+		return expectations;
+	}
+	expectations.result.emplace_back(param);
+
+	while (accept(Token::Comma, true))
+		expectations.result.emplace_back(parseParameter());
+
+	/// We have always one virtual parameter in the parameter list.
+	/// If its type is FAILURE, the expected result is also a REVERT etc.
+	if (expectations.result.at(0).abiType.type != ABIType::Failure)
+		expectations.failure = false;
+	return expectations;
+}
+
+Parameter TestFileParser::parseParameter()
+{
+	Parameter parameter;
+	if (accept(Token::Newline, true))
+		parameter.format.newline = true;
+	auto literal = parseABITypeLiteral();
+	parameter.rawBytes = literal.first;
+	parameter.abiType = literal.second;
+	return parameter;
+}
+
+pair<bytes, ABIType> TestFileParser::parseABITypeLiteral()
+{
+	try
+	{
+		u256 number{0};
+		ABIType abiType{ABIType::None, 0};
+
+		if (accept(Token::Sub))
+		{
+			abiType = ABIType{ABIType::SignedDec, 32};
+			expect(Token::Sub);
+			number = convertNumber(parseNumber()) * -1;
+		}
+		else
+		{
+			if (accept(Token::Number))
+			{
+				abiType = ABIType{ABIType::UnsignedDec, 32};
+				number = convertNumber(parseNumber());
+			}
+			else if (accept(Token::Failure, true))
+			{
+				abiType = ABIType{ABIType::Failure, 0};
+				return make_pair(bytes{}, abiType);
+			}
+		}
+		return make_pair(toBigEndian(number), abiType);
+	}
+	catch (std::exception const&)
+	{
+		throw Error(Error::Type::ParserError, "Number encoding invalid.");
+	}
+}
+
+string TestFileParser::parseIdentifierOrTuple()
+{
+	string identOrTuple;
+
+	if (accept(Token::Identifier))
+	{
+		identOrTuple = m_scanner.currentLiteral();
+		expect(Token::Identifier);
+		return identOrTuple;
+	}
+	expect(Token::LParen);
+	identOrTuple += formatToken(Token::LParen);
+	identOrTuple += parseIdentifierOrTuple();
+
+	while (accept(Token::Comma))
+	{
+		identOrTuple += formatToken(Token::Comma);
+		expect(Token::Comma);
+		identOrTuple += parseIdentifierOrTuple();
+	}
+	expect(Token::RParen);
+	identOrTuple += formatToken(Token::RParen);
+	return identOrTuple;
+}
+
+string TestFileParser::parseComment()
+{
+	string comment = m_scanner.currentLiteral();
+	if (accept(Token::Comment, true))
+		return comment;
+	return string{};
+}
+
+string TestFileParser::parseNumber()
+{
+	string literal = m_scanner.currentLiteral();
+	expect(Token::Number);
+	return literal;
+}
+
+u256 TestFileParser::convertNumber(string const& _literal)
+{
+	try {
+		return u256{_literal};
+	}
+	catch (std::exception const&)
+	{
+		throw Error(Error::Type::ParserError, "Number encoding invalid.");
+	}
+}
+
+void TestFileParser::Scanner::readStream(istream& _stream)
+{
+	std::string line;
+	while (std::getline(_stream, line))
+		m_line += line;
+	m_char = m_line.begin();
+}
+
+void TestFileParser::Scanner::scanNextToken()
+{
+	// Make code coverage happy.
+	assert(formatToken(Token::NUM_TOKENS) == "");
+
+	auto detectKeyword = [](std::string const& _literal = "") -> TokenDesc {
+		if (_literal == "ether") return TokenDesc{Token::Ether, _literal};
+		if (_literal == "FAILURE") return TokenDesc{Token::Failure, _literal};
+		return TokenDesc{Token::Identifier, _literal};
+	};
+
+	auto selectToken = [this](Token _token, std::string const& _literal = "") -> TokenDesc {
+		advance();
+		return make_pair(_token, !_literal.empty() ? _literal : formatToken(_token));
+	};
+
+	TokenDesc token = make_pair(Token::Unknown, "");
+	do
+	{
+		switch(current())
+		{
+		case '/':
+			advance();
+			if (current() == '/')
+				token = selectToken(Token::Newline);
+			else
+				token = selectToken(Token::Invalid);
+			break;
+		case '-':
+			if (peek() == '>')
+			{
+				advance();
+				token = selectToken(Token::Arrow);
+			}
+			else
+				token = selectToken(Token::Sub);
+			break;
+		case ':':
+			token = selectToken(Token::Colon);
+			break;
+		case '#':
+			token = selectToken(Token::Comment, scanComment());
+			break;
+		case ',':
+			token = selectToken(Token::Comma);
+			break;
+		case '(':
+			token = selectToken(Token::LParen);
+			break;
+		case ')':
+			token = selectToken(Token::RParen);
+			break;
+		default:
+			if (isIdentifierStart(current()))
+			{
+				TokenDesc detectedToken = detectKeyword(scanIdentifierOrKeyword());
+				token = selectToken(detectedToken.first, detectedToken.second);
+			}
+			else if (isdigit(current()))
+				token = selectToken(Token::Number, scanNumber());
+			else if (isspace(current()))
+				token = selectToken(Token::Whitespace);
+			else if (isEndOfLine())
+				token = selectToken(Token::EOS);
+			break;
+		}
+	}
+	while (token.first == Token::Whitespace);
+	m_currentToken = token;
+}
+
+string TestFileParser::Scanner::scanComment()
+{
+	string comment;
+	advance();
+
+	while (current() != '#')
+	{
+		comment += current();
+		advance();
+	}
+	return comment;
+}
+
+string TestFileParser::Scanner::scanIdentifierOrKeyword()
+{
+	string identifier;
+	identifier += current();
+	while (isIdentifierPart(peek()))
+	{
+		advance();
+		identifier += current();
+	}
+	return identifier;
+}
+
+string TestFileParser::Scanner::scanNumber()
+{
+	string number;
+	number += current();
+	while (isdigit(peek()))
+	{
+		advance();
+		number += current();
+	}
+	return number;
+}

test/libsolidity/util/TestFileParser.h

@@ -0,0 +1,359 @@
+/*
+	This file is part of solidity.
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <libdevcore/CommonData.h>
+#include <libsolidity/ast/Types.h>
+#include <liblangutil/Exceptions.h>
+
+#include <iosfwd>
+#include <numeric>
+#include <stdexcept>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+/**
+ * All soltest tokens.
+ */
+#define SOLT_TOKEN_LIST(T, K)      \
+	T(Unknown, "unknown", 0)       \
+	T(Invalid, "invalid", 0)       \
+	T(EOS, "EOS", 0)               \
+	T(Whitespace, "_", 0)          \
+	/* punctuations */             \
+	T(LParen, "(", 0)              \
+	T(RParen, ")", 0)              \
+	T(LBrack, "[", 0)              \
+	T(RBrack, "]", 0)              \
+	T(LBrace, "{", 0)              \
+	T(RBrace, "}", 0)              \
+	T(Sub,    "-", 0)              \
+	T(Colon,  ":", 0)              \
+	T(Comma,  ",", 0)              \
+	T(Period, ".", 0)              \
+	T(Arrow, "->", 0)              \
+	T(Newline, "//", 0)            \
+	/* Literals & identifier */    \
+	T(Comment, "#", 0)             \
+	T(Number, "number", 0)         \
+	T(Identifier, "identifier", 0) \
+	/* type keywords */            \
+	K(Ether, "ether", 0)           \
+	/* special keywords */         \
+	K(Failure, "FAILURE", 0)       \
+
+namespace soltest
+{
+	enum class Token : unsigned int {
+	#define T(name, string, precedence) name,
+		SOLT_TOKEN_LIST(T, T)
+		NUM_TOKENS
+	#undef T
+	};
+
+	/// Prints a friendly string representation of \param _token.
+	inline std::string formatToken(Token _token)
+	{
+		switch (_token)
+		{
+	#define T(name, string, precedence) case Token::name: return string;
+			SOLT_TOKEN_LIST(T, T)
+	#undef T
+			default: // Token::NUM_TOKENS:
+				return "";
+		}
+	}
+}
+
+
+
+/**
+ * The purpose of the ABI type is the storage of type information
+ * retrieved while parsing a test. This information is used
+ * for the conversion of human-readable function arguments and
+ * return values to `bytes` and vice-versa.
+ * Defaults to None, a 0-byte representation. 0-bytes
+ * can also be interpreted as Failure, which means
+ * either a REVERT or another EVM failure.
+ */
+struct ABIType
+{
+	enum Type {
+		UnsignedDec,
+		SignedDec,
+		Failure,
+		None
+	};
+	Type type = ABIType::None;
+	size_t size = 0;
+};
+
+/**
+ * Helper that can hold format information retrieved
+ * while scanning through a parameter list in soltest.
+ */
+struct FormatInfo
+{
+	bool newline;
+};
+
+/**
+ * Parameter abstraction used for the encoding and decoding of
+ * function parameter and expectation / return value lists.
+ * A parameter list is usually a comma-separated list of literals.
+ * It should not be possible to call create a parameter holding
+ * an identifier, but if so, the ABI type would be invalid.
+ */
+struct Parameter
+{
+	/// ABI encoded / decoded `bytes` of values.
+	/// These `bytes` are used to pass values to function calls
+	/// and also to store expected return vales. These are
+	/// compared to the actual result of a function call
+	/// and used for validating it.
+	bytes rawBytes;
+	/// Types that were used to encode `rawBytes`. Expectations
+	/// are usually comma separated literals. Their type is auto-
+	/// detected and retained in order to format them later on.
+	ABIType abiType;
+	/// Format info attached to the parameter. It handles newlines given
+	/// in the declaration of it.
+	FormatInfo format;
+};
+using ParameterList = std::vector<Parameter>;
+
+/**
+ * Represents the expected result of a function call after it has been executed. This may be a single
+ * return value or a comma-separated list of return values. It also contains the detected input
+ * formats used to convert the values to `bytes` needed for the comparison with the actual result
+ * of a call. In addition to that, it also stores the expected transaction status.
+ * An optional comment can be assigned.
+ */
+struct FunctionCallExpectations
+{
+	/// Representation of the comma-separated (or empty) list of expected result values
+	/// attached to the function call object. It is checked against the actual result of
+	/// a function call when used in test framework.
+	ParameterList result;
+	/// Expected status of the transaction. It can be either
+	/// a REVERT or a different EVM failure (e.g. out-of-gas).
+	bool failure = true;
+	/// A Comment that can be attached to the expectations,
+	/// that is retained and can be displayed.
+	std::string comment;
+	/// ABI encoded `bytes` of parsed expected return values. It is checked
+	/// against the actual result of a function call when used in test framework.
+	bytes rawBytes() const
+	{
+		bytes raw;
+		for (auto const& param: result)
+			raw += param.rawBytes;
+		return raw;
+	}
+};
+
+/**
+ * Represents the arguments passed to a function call. This can be a single
+ * argument or a comma-separated list of arguments. It also contains the detected input
+ * formats used to convert the arguments to `bytes` needed for the call.
+ * An optional comment can be assigned.
+ */
+struct FunctionCallArgs
+{
+	/// Types that were used to encode `rawBytes`. Parameters
+	/// are usually comma separated literals. Their type is auto-
+	/// detected and retained in order to format them later on.
+	ParameterList parameters;
+	/// A Comment that can be attached to the expectations,
+	/// that is retained and can be displayed.
+	std::string comment;
+	/// ABI encoded `bytes` of parsed parameters. These `bytes`
+	/// passed to the function call.
+	bytes rawBytes() const
+	{
+		bytes raw;
+		for (auto const& param: parameters)
+			raw += param.rawBytes;
+		return raw;
+	}
+};
+
+/**
+ * Represents a function call read from an input stream. It contains the signature, the
+ * arguments, an optional ether value and an expected execution result.
+ */
+struct FunctionCall
+{
+	/// Signature of the function call, e.g. `f(uint256, uint256)`.
+	std::string signature;
+	/// Optional `ether` value that can be send with the call.
+	u256 value;
+	/// Object that holds all function parameters in their `bytes`
+	/// representations given by the contract ABI.
+	FunctionCallArgs arguments;
+	/// Object that holds all function call expectation in
+	/// their `bytes` representations given by the contract ABI.
+	/// They are checked against the actual results and their
+	/// `bytes` representation, as well as the transaction status.
+	FunctionCallExpectations expectations;
+	/// single / multi-line mode will be detected as follows:
+	/// every newline (//) in source results in a function call
+	/// that has its display mode set to multi-mode. Function and
+	/// result parameter lists are an exception: a single parameter
+	/// stores a format information that contains a newline definition.
+	enum DisplayMode {
+		SingleLine,
+		MultiLine
+	};
+	DisplayMode displayMode = DisplayMode::SingleLine;
+};
+
+/**
+ * Class that is able to parse an additional and well-formed comment section in a Solidity
+ * source file used by the file-based unit test environment. For now, it parses function
+ * calls and their expected result after the call was made.
+ *
+ * - Function calls defined in blocks:
+ * // f(uint256, uint256): 1, 1 # Signature and comma-separated list of arguments #
+ * // -> 1, 1                   # Expected result value #
+ * // g(), 2 ether              # (Optional) Ether to be send with the call #
+ * // -> 2, 3
+ * // h(uint256), 1 ether: 42
+ * // -> FAILURE                # If REVERT or other EVM failure was detected #
+ * ...
+ */
+class TestFileParser
+{
+public:
+	/// Constructor that takes an input stream \param _stream to operate on
+	/// and creates the internal scanner.
+	TestFileParser(std::istream& _stream): m_scanner(_stream) {}
+
+	/// Parses function calls blockwise and returns a list of function calls found.
+	/// Throws an exception if a function call cannot be parsed because of its
+	/// incorrect structure, an invalid or unsupported encoding
+	/// of its arguments or expected results.
+	std::vector<FunctionCall> parseFunctionCalls();
+
+private:
+	using Token = soltest::Token;
+	/**
+	 * Token scanner that is used internally to abstract away character traversal.
+	 */
+	class Scanner
+	{
+	public:
+		/// Constructor that takes an input stream \param _stream to operate on.
+		/// It reads all lines into one single line, keeping the newlines.
+		Scanner(std::istream& _stream) { readStream(_stream); }
+
+		/// Reads input stream into a single line and resets the current iterator.
+		void readStream(std::istream& _stream);
+
+		/// Reads character stream and creates token.
+		void scanNextToken();
+
+		soltest::Token currentToken() { return m_currentToken.first; }
+		std::string currentLiteral() { return m_currentToken.second; }
+
+		std::string scanComment();
+		std::string scanIdentifierOrKeyword();
+		std::string scanNumber();
+
+	private:
+		using TokenDesc = std::pair<Token, std::string>;
+
+		/// Advances current position in the input stream.
+		void advance() { ++m_char; }
+		/// Returns the current character.
+		char current() const { return *m_char; }
+		/// Peeks the next character.
+		char peek() const { auto it = m_char; return *(it + 1); }
+		/// Returns true if the end of a line is reached, false otherwise.
+		bool isEndOfLine() const { return m_char == m_line.end(); }
+
+		std::string m_line;
+		std::string::iterator m_char;
+
+		std::string m_currentLiteral;
+
+		TokenDesc m_currentToken;
+	};
+
+	bool accept(soltest::Token _token, bool const _expect = false);
+	bool expect(soltest::Token _token, bool const _advance = true);
+
+	/// Parses a function call signature in the form of f(uint256, ...).
+	std::string parseFunctionSignature();
+
+	/// Parses the optional ether value that can be passed alongside the
+	/// function call arguments. Throws an InvalidEtherValueEncoding exception
+	/// if given value cannot be converted to `u256`.
+	u256 parseFunctionCallValue();
+
+	/// Parses a comma-separated list of arguments passed with a function call.
+	/// Does not check for a potential mismatch between the signature and the number
+	/// or types of arguments.
+	FunctionCallArgs parseFunctionCallArguments();
+
+	/// Parses the expected result of a function call execution.
+	FunctionCallExpectations parseFunctionCallExpectations();
+
+	/// Parses the next parameter in a comma separated list.
+	/// Takes a newly parsed, and type-annotated `bytes` argument,
+	/// appends it to the internal `bytes` buffer of the parameter. It can also
+	/// store newlines found in the source, that are needed to
+	/// format input and output of the interactive update.
+	Parameter parseParameter();
+
+	/// Parses and converts the current literal to its byte representation and
+	/// preserves the chosen ABI type. Based on that type information, the driver of
+	/// this parser can format arguments, expectations and results. Supported types:
+	/// - unsigned and signed decimal number literals.
+	/// Returns invalid ABI type for empty literal. This is needed in order
+	/// to detect empty expectations. Throws a ParserError if data is encoded incorrectly or
+	/// if data type is not supported.
+	std::pair<bytes, ABIType> parseABITypeLiteral();
+
+	/// Recursively parses an identifier or a tuple definition that contains identifiers
+	/// and / or parentheses like `((uint, uint), (uint, (uint, uint)), uint)`.
+	std::string parseIdentifierOrTuple();
+
+	/// Parses a comment that is defined like this:
+	/// # A nice comment. #
+	std::string parseComment();
+
+	/// Parses the current number literal.
+	std::string parseNumber();
+
+	/// Tries to convert \param _literal to `uint256` and throws if
+	/// conversion fails.
+	u256 convertNumber(std::string const& _literal);
+
+	/// A scanner instance
+	Scanner m_scanner;
+};
+
+}
+}
+}

test/libsolidity/util/TestFileParserTests.cpp

@@ -0,0 +1,531 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * Unit tests for Solidity's test expectation parser.
+ */
+
+#include <functional>
+#include <string>
+#include <tuple>
+#include <boost/test/unit_test.hpp>
+#include <liblangutil/Exceptions.h>
+#include <test/ExecutionFramework.h>
+
+#include <test/libsolidity/util/TestFileParser.h>
+
+using namespace std;
+using namespace dev::test;
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+using fmt = ExecutionFramework;
+using Mode = FunctionCall::DisplayMode;
+
+vector<FunctionCall> parse(string const& _source)
+{
+	istringstream stream{_source, ios_base::out};
+	TestFileParser parser{stream};
+	return parser.parseFunctionCalls();
+}
+
+void testFunctionCall(
+		FunctionCall const& _call,
+		FunctionCall::DisplayMode _mode,
+		string _signature = "",
+		bool _failure = true,
+		bytes _arguments = bytes{},
+		bytes _expectations = bytes{},
+		u256 _value = 0,
+		string _argumentComment = "",
+		string _expectationComment = ""
+)
+{
+	BOOST_REQUIRE_EQUAL(_call.expectations.failure, _failure);
+	BOOST_REQUIRE_EQUAL(_call.signature, _signature);
+	ABI_CHECK(_call.arguments.rawBytes(), _arguments);
+	ABI_CHECK(_call.expectations.rawBytes(), _expectations);
+	BOOST_REQUIRE_EQUAL(_call.displayMode, _mode);
+	BOOST_REQUIRE_EQUAL(_call.value, _value);
+	BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment);
+	BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment);
+}
+
+BOOST_AUTO_TEST_SUITE(TestFileParserTest)
+
+BOOST_AUTO_TEST_CASE(smoke_test)
+{
+	char const* source = R"()";
+	BOOST_REQUIRE_EQUAL(parse(source).size(), 0);
+}
+
+BOOST_AUTO_TEST_CASE(call_succees)
+{
+	char const* source = R"(
+		// success() ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(calls.at(0), Mode::SingleLine, "success()", false);
+}
+
+BOOST_AUTO_TEST_CASE(non_existent_call_revert_single_line)
+{
+	char const* source = R"(
+		// i_am_not_there() -> FAILURE
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(calls.at(0), Mode::SingleLine, "i_am_not_there()", true);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_success)
+{
+	char const* source = R"(
+		// f(uint256): 1
+		// ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(calls.at(0), Mode::MultiLine, "f(uint256)", false, fmt::encodeArgs(u256{1}));
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_comments_success)
+{
+	char const* source = R"(
+		// f(uint256, uint256): 1, 1
+		// ->
+		// # This call should not return a value, but still succeed. #
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::MultiLine,
+		"f(uint256,uint256)",
+		false,
+		fmt::encodeArgs(1, 1),
+		fmt::encodeArgs(),
+		0,
+		"",
+		" This call should not return a value, but still succeed. "
+	);
+}
+
+BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success)
+{
+	char const* source = R"(
+		// f(uint256): 1 -> # f(uint256) does not return a value. #
+		// f(uint256): 1 -> 1
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+
+	testFunctionCall(
+		calls.at(0),
+		Mode::SingleLine,
+		"f(uint256)",
+		false,
+		fmt::encodeArgs(1),
+		fmt::encodeArgs(),
+		0,
+		"",
+		" f(uint256) does not return a value. "
+	);
+	testFunctionCall(calls.at(1), Mode::SingleLine, "f(uint256)", false, fmt::encode(1), fmt::encode(1));
+}
+
+BOOST_AUTO_TEST_CASE(multiple_single_line)
+{
+	char const* source = R"(
+		// f(uint256): 1 -> 1
+		// g(uint256): 1 ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+
+	testFunctionCall(calls.at(0), Mode::SingleLine, "f(uint256)", false, fmt::encodeArgs(1), fmt::encodeArgs(1));
+	testFunctionCall(calls.at(1), Mode::SingleLine, "g(uint256)", false, fmt::encodeArgs(1));
+}
+
+BOOST_AUTO_TEST_CASE(multiple_single_line_swapped)
+{
+	char const* source = R"(
+		// f(uint256): 1 ->
+		// g(uint256): 1 -> 1
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+
+	testFunctionCall(calls.at(0), Mode::SingleLine, "f(uint256)", false, fmt::encodeArgs(1));
+	testFunctionCall(calls.at(1), Mode::SingleLine, "g(uint256)", false, fmt::encodeArgs(1), fmt::encodeArgs(1));
+
+}
+
+BOOST_AUTO_TEST_CASE(non_existent_call_revert)
+{
+	char const* source = R"(
+		// i_am_not_there()
+		// -> FAILURE
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(calls.at(0), Mode::MultiLine, "i_am_not_there()", true);
+}
+
+BOOST_AUTO_TEST_CASE(call_expectations_empty_single_line)
+{
+	char const* source = R"(
+		// _exp_() ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(calls.at(0), Mode::SingleLine, "_exp_()", false);
+}
+
+BOOST_AUTO_TEST_CASE(call_expectations_empty_multiline)
+{
+	char const* source = R"(
+		// _exp_()
+		// ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(calls.at(0), Mode::MultiLine, "_exp_()", false);
+}
+
+BOOST_AUTO_TEST_CASE(call_comments)
+{
+	char const* source = R"(
+		// f() # Parameter comment # -> 1 # Expectation comment #
+		// f() # Parameter comment #
+		// -> 1 # Expectation comment #
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+	testFunctionCall(
+		calls.at(0),
+		Mode::SingleLine,
+		"f()",
+		false,
+		fmt::encodeArgs(),
+		fmt::encodeArgs(1),
+		0,
+		" Parameter comment ",
+		" Expectation comment "
+	);
+	testFunctionCall(
+		calls.at(1),
+		Mode::MultiLine,
+		"f()",
+		false,
+		fmt::encodeArgs(),
+		fmt::encodeArgs(1),
+		0,
+		" Parameter comment ",
+		" Expectation comment "
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments)
+{
+	char const* source = R"(
+		// f(uint256), 314 ether: 5 # optional ether value #
+		// -> 4
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::MultiLine,
+		"f(uint256)",
+		false,
+		fmt::encodeArgs(5),
+		fmt::encodeArgs(4),
+		314,
+		" optional ether value "
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_tuple)
+{
+	char const* source = R"(
+		// f((uint256, bytes32), uint256) ->
+		// f((uint8), uint8) ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+	testFunctionCall(calls.at(0), Mode::SingleLine, "f((uint256,bytes32),uint256)", false);
+	testFunctionCall(calls.at(1), Mode::SingleLine, "f((uint8),uint8)", false);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_tuple_of_tuples)
+{
+	char const* source = R"(
+		// f(((uint256, bytes32), bytes32), uint256)
+		// # f(S memory s, uint256 b) #
+		// ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::MultiLine,
+		"f(((uint256,bytes32),bytes32),uint256)",
+		false,
+		fmt::encodeArgs(),
+		fmt::encodeArgs(),
+		0,
+		" f(S memory s, uint256 b) "
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_recursive_tuples)
+{
+	char const* source = R"(
+		// f(((((bytes, bytes, bytes), bytes), bytes), bytes), bytes) ->
+		// f(((((bytes, bytes, (bytes)), bytes), bytes), (bytes, bytes)), (bytes, bytes)) ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+	testFunctionCall(
+		calls.at(0),
+		Mode::SingleLine,
+		"f(((((bytes,bytes,bytes),bytes),bytes),bytes),bytes)",
+		false
+	);
+	testFunctionCall(
+		calls.at(1),
+		Mode::SingleLine,
+		"f(((((bytes,bytes,(bytes)),bytes),bytes),(bytes,bytes)),(bytes,bytes))",
+		false
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_mismatch)
+{
+	char const* source = R"(
+		// f(uint256):
+		// 1, 2
+		// # This only throws at runtime #
+		// -> 1
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::MultiLine,
+		"f(uint256)",
+		false,
+		fmt::encodeArgs(1, 2),
+		fmt::encodeArgs(1),
+		0,
+		" This only throws at runtime "
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_multiple_arguments)
+{
+	char const* source = R"(
+		// test(uint256, uint256):
+		// 1,
+		// 2
+		// -> 1,
+		// 1
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::MultiLine,
+		"test(uint256,uint256)",
+		false,
+		fmt::encodeArgs(1, 2),
+		fmt::encodeArgs(1, 1)
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format)
+{
+	char const* source = R"(
+		// test(uint256, uint256), 314 ether:
+		// 1, -2
+		// -> -1, 2
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::MultiLine,
+		"test(uint256,uint256)",
+		false,
+		fmt::encodeArgs(1, -2),
+		fmt::encodeArgs(-1, 2),
+		314
+	);
+}
+
+BOOST_AUTO_TEST_CASE(call_signature)
+{
+	char const* source = R"(
+		// f(uint256, uint8, string) -> FAILURE
+		// f(invalid, xyz, foo) -> FAILURE
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
+	testFunctionCall(calls.at(0), Mode::SingleLine, "f(uint256,uint8,string)", true);
+	testFunctionCall(calls.at(1), Mode::SingleLine, "f(invalid,xyz,foo)", true);
+}
+
+BOOST_AUTO_TEST_CASE(call_newline_invalid)
+{
+	char const* source = R"(
+		/
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_invalid)
+{
+	char const* source = R"(
+		/ f() ->
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_signature_invalid)
+{
+	char const* source = R"(
+		// f(uint8,) -> FAILURE
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_tuple_invalid)
+{
+	char const* source = R"(
+		// f((uint8,) -> FAILURE
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_tuple_invalid_empty)
+{
+	char const* source = R"(
+		// f(uint8, ()) -> FAILURE
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_tuple_invalid_parantheses)
+{
+	char const* source = R"(
+		// f((uint8,() -> FAILURE
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_expectations_missing)
+{
+	char const* source = R"(
+		// f())";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_ether_value_expectations_missing)
+{
+	char const* source = R"(
+		// f(), 0)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_invalid)
+{
+	char const* source = R"(
+		// f(uint256): abc -> 1
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_invalid_decimal)
+{
+	char const* source = R"(
+		// sig(): 0.h3 ->
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_ether_value_invalid)
+{
+	char const* source = R"(
+		// f(uint256), abc : 1 -> 1
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_ether_value_invalid_decimal)
+{
+	char const* source = R"(
+		// sig(): 0.1hd ether ->
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_ether_type_invalid)
+{
+	char const* source = R"(
+		// f(uint256), 2 btc : 1 -> 1
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_colon)
+{
+	char const* source = R"(
+		// h256():
+		// -> 1
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arguments_newline_colon)
+{
+	char const* source = R"(
+		// h256()
+		// :
+		// -> 1
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_CASE(call_arrow_missing)
+{
+	char const* source = R"(
+		// h256()
+	)";
+	BOOST_REQUIRE_THROW(parse(source), langutil::Error);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+}
+}
+}