Merge pull request #5872 from ethereum/semantic-tests-split

Semantic test infrastructure

test/ExecutionFramework.cpp

@@ -49,8 +49,13 @@ string getIPCSocketPath()
 
 }
 
-ExecutionFramework::ExecutionFramework() :
-	m_rpc(RPCSession::instance(getIPCSocketPath())),
+ExecutionFramework::ExecutionFramework():
+	ExecutionFramework(getIPCSocketPath())
+{
+}
+
+ExecutionFramework::ExecutionFramework(string const& _ipcPath):
+	m_rpc(RPCSession::instance(_ipcPath)),
 	m_evmVersion(dev::test::Options::get().evmVersion()),
 	m_optimize(dev::test::Options::get().optimize),
 	m_showMessages(dev::test::Options::get().showMessages),

test/ExecutionFramework.h

@@ -53,6 +53,7 @@ class ExecutionFramework
 
 public:
 	ExecutionFramework();
+	explicit ExecutionFramework(std::string const& _ipcPath);
 	virtual ~ExecutionFramework() = default;
 
 	virtual bytes const& compileAndRunWithoutCheck(

test/InteractiveTests.h

@@ -20,6 +20,7 @@
 #include <test/TestCase.h>
 #include <test/libsolidity/ASTJSONTest.h>
 #include <test/libsolidity/SyntaxTest.h>
+#include <test/libsolidity/SemanticTest.h>
 #include <test/libsolidity/SMTCheckerJSONTest.h>
 #include <test/libyul/YulOptimizerTest.h>
 #include <test/libyul/ObjectCompilerTest.h>
@@ -52,6 +53,7 @@ Testsuite const g_interactiveTestsuites[] = {
 	{"Yul Optimizer",       "libyul",      "yulOptimizerTests",   false, false, &yul::test::YulOptimizerTest::create},
 	{"Yul Object Compiler", "libyul",      "objectCompiler",      false, false, &yul::test::ObjectCompilerTest::create},
 	{"Syntax",              "libsolidity", "syntaxTests",         false, false, &SyntaxTest::create},
+	{"Semantic",            "libsolidity", "semanticTests",       false, true,  &SemanticTest::create},
 	{"JSON AST",            "libsolidity", "ASTJSON",             false, false, &ASTJSONTest::create},
 	{"SMT Checker",         "libsolidity", "smtCheckerTests",     true,  false, &SyntaxTest::create},
 	{"SMT Checker JSON",    "libsolidity", "smtCheckerTestsJSON", true,  false, &SMTCheckerTest::create}

test/RPCSession.cpp

@@ -139,9 +139,16 @@ string IPCSocket::sendRequest(string const& _req)
 
 RPCSession& RPCSession::instance(string const& _path)
 {
-	static RPCSession session(_path);
-	BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path);
-	return session;
+	try
+	{
+		static RPCSession session(_path);
+		BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path);
+		return session;
+	}
+	catch (std::exception const&)
+	{
+		BOOST_THROW_EXCEPTION(std::runtime_error("Error creating RPC session for socket: " + _path));
+	}
 }
 
 string RPCSession::eth_getCode(string const& _address, string const& _blockNumber)

test/TestCase.h

@@ -30,11 +30,25 @@ namespace solidity
 namespace test
 {
 
+#define soltestAssert(CONDITION, DESCRIPTION) \
+	do \
+	{ \
+		if (!(CONDITION)) \
+			BOOST_THROW_EXCEPTION(runtime_error(DESCRIPTION)); \
+	} \
+	while (false)
+
 /** Common superclass of SyntaxTest and SemanticsTest. */
 class TestCase
 {
 public:
-	using TestCaseCreator = std::unique_ptr<TestCase>(*)(std::string const&);
+	struct Config
+	{
+		std::string filename;
+		std::string ipcPath;
+	};
+
+	using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
 
 	virtual ~TestCase() = default;
 

test/boostTest.cpp

@@ -74,11 +74,13 @@ int registerTests(
 	boost::unit_test::test_suite& _suite,
 	boost::filesystem::path const& _basepath,
 	boost::filesystem::path const& _path,
+	std::string const& _ipcPath,
 	TestCase::TestCaseCreator _testCaseCreator
 )
 {
 	int numTestsAdded = 0;
 	fs::path fullpath = _basepath / _path;
+	TestCase::Config config{fullpath.string(), _ipcPath};
 	if (fs::is_directory(fullpath))
 	{
 		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
@@ -87,7 +89,7 @@ int registerTests(
 			fs::directory_iterator()
 		))
 			if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
-				numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _testCaseCreator);
+				numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _ipcPath, _testCaseCreator);
 		_suite.add(sub_suite);
 	}
 	else
@@ -96,13 +98,13 @@ int registerTests(
 
 		filenames.emplace_back(new string(_path.string()));
 		_suite.add(make_test_case(
-			[fullpath, _testCaseCreator]
+			[config, _testCaseCreator]
 			{
 				BOOST_REQUIRE_NO_THROW({
 					try
 					{
 						stringstream errorStream;
-						if (!_testCaseCreator(fullpath.string())->run(errorStream))
+						if (!_testCaseCreator(config)->run(errorStream))
 							BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
 					}
 					catch (boost::exception const& _e)
@@ -142,6 +144,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 			master,
 			options.testPath / ts.path,
 			ts.subpath,
+			options.ipcPath,
 			ts.testCaseCreator
 		) > 0, std::string("no ") + ts.title + " tests found");
 	}

test/libsolidity/ASTJSONTest.h

@@ -35,8 +35,8 @@ namespace test
 class ASTJSONTest: public TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
-	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_filename)); }
+	static std::unique_ptr<TestCase> create(Config const& _config)
+	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
 	ASTJSONTest(std::string const& _filename);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;

test/libsolidity/SMTCheckerJSONTest.h

@@ -33,9 +33,9 @@ namespace test
 class SMTCheckerTest: public SyntaxTest
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new SMTCheckerTest(_filename));
+		return std::unique_ptr<TestCase>(new SMTCheckerTest(_config.filename));
 	}
 	SMTCheckerTest(std::string const& _filename);
 

test/libsolidity/SemanticTest.cpp

@@ -0,0 +1,275 @@
+/*
+	This file is part of solidity.
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SemanticTest.h>
+#include <test/Options.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/trim.hpp>
+#include <boost/throw_exception.hpp>
+
+#include <algorithm>
+#include <cctype>
+#include <fstream>
+#include <memory>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace dev::formatting;
+using namespace std;
+using namespace boost;
+using namespace boost::algorithm;
+using namespace boost::unit_test;
+namespace fs = boost::filesystem;
+
+namespace
+{
+	using FunctionCallTest = SemanticTest::FunctionCallTest;
+	using FunctionCall = dev::solidity::test::FunctionCall;
+	using ParamList = dev::solidity::test::ParameterList;
+
+
+	string formatBytes(bytes const& _bytes, ParamList const& _params)
+	{
+		stringstream resultStream;
+		if (_bytes.empty())
+			return {};
+		auto it = _bytes.begin();
+		for (auto const& param: _params)
+		{
+			long offset = static_cast<long>(param.abiType.size);
+			auto offsetIter = it + offset;
+			soltestAssert(offsetIter <= _bytes.end(), "Byte range can not be extended past the end of given bytes.");
+
+			bytes byteRange{it, offsetIter};
+			switch (param.abiType.type)
+			{
+			case ABIType::SignedDec:
+				if (*byteRange.begin() & 0x80)
+					resultStream << u2s(fromBigEndian<u256>(byteRange));
+				else
+					resultStream << fromBigEndian<u256>(byteRange);
+				break;
+			case ABIType::UnsignedDec:
+				// Check if the detected type was wrong and if this could
+				// be signed. If an unsigned was detected in the expectations,
+				// but the actual result returned a signed, it would be formatted
+				// incorrectly.
+				if (*byteRange.begin() & 0x80)
+					resultStream << u2s(fromBigEndian<u256>(byteRange));
+				else
+					resultStream << fromBigEndian<u256>(byteRange);
+				break;
+			case ABIType::Failure:
+				break;
+			case ABIType::None:
+				break;
+			}
+			it += offset;
+			if (it != _bytes.end() && !(param.abiType.type == ABIType::None))
+				resultStream << ", ";
+		}
+		soltestAssert(it == _bytes.end(), "Parameter encoding too short for the given byte range.");
+		return resultStream.str();
+	}
+
+	string formatRawArguments(ParamList const& _params, string const& _linePrefix = "")
+	{
+		stringstream resultStream;
+		for (auto const& param: _params)
+		{
+			if (param.format.newline)
+				resultStream << endl << _linePrefix << "//";
+			resultStream << " " << param.rawString;
+			if (&param != &_params.back())
+				resultStream << ",";
+		}
+		return resultStream.str();
+	}
+
+	string formatFunctionCallTest(
+		FunctionCallTest const& _test,
+		string const& _linePrefix = "",
+		bool const _renderResult = false,
+		bool const _highlight = false
+	)
+	{
+		using namespace soltest;
+		using Token = soltest::Token;
+
+		stringstream _stream;
+		FunctionCall call = _test.call;
+		bool highlight = !_test.matchesExpectation() && _highlight;
+
+		auto formatOutput = [&](bool const _singleLine)
+		{
+			string ws = " ";
+			string arrow = formatToken(Token::Arrow);
+			string colon = formatToken(Token::Colon);
+			string comma = formatToken(Token::Comma);
+			string comment = formatToken(Token::Comment);
+			string ether = formatToken(Token::Ether);
+			string newline = formatToken(Token::Newline);
+			string failure = formatToken(Token::Failure);
+
+			/// Prints the function signature. This is the same independent from the display-mode.
+			_stream << _linePrefix << newline << ws << call.signature;
+			if (call.value > u256(0))
+				_stream << comma << ws << call.value << ws << ether;
+			if (!call.arguments.rawBytes().empty())
+			{
+				string output = formatRawArguments(call.arguments.parameters, _linePrefix);
+				_stream << colon << output;
+			}
+
+			/// Prints comments on the function parameters and the arrow taking
+			/// the display-mode into account.
+			if (_singleLine)
+			{
+				if (!call.arguments.comment.empty())
+					_stream << ws << comment << call.arguments.comment << comment;
+				_stream << ws << arrow << ws;
+			}
+			else
+			{
+				_stream << endl << _linePrefix << newline << ws;
+				if (!call.arguments.comment.empty())
+				{
+					 _stream << comment << call.arguments.comment << comment;
+					 _stream << endl << _linePrefix << newline << ws;
+				}
+				_stream << arrow << ws;
+			}
+
+			/// Print either the expected output or the actual result output
+			string result;
+			if (!_renderResult)
+			{
+				bytes output = call.expectations.rawBytes();
+				bool const isFailure = call.expectations.failure;
+				result = isFailure ? failure : formatBytes(output, call.expectations.result);
+			}
+			else
+			{
+				bytes output = _test.rawBytes;
+				bool const isFailure = _test.failure;
+				result = isFailure ? failure : formatBytes(output, call.expectations.result);
+			}
+			AnsiColorized(_stream, highlight, {RED_BACKGROUND}) << result;
+
+			/// Print comments on expectations taking the display-mode into account.
+			if (_singleLine)
+			{
+				if (!call.expectations.comment.empty())
+					_stream << ws << comment << call.expectations.comment << comment;
+			}
+			else
+			{
+				if (!call.expectations.comment.empty())
+				{
+					_stream << endl << _linePrefix << newline << ws;
+					_stream << comment << call.expectations.comment << comment;
+				}
+			}
+		};
+
+		if (call.displayMode == FunctionCall::DisplayMode::SingleLine)
+			formatOutput(true);
+		else
+			formatOutput(false);
+		_stream << endl;
+
+		return _stream.str();
+	}
+}
+
+SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
+	SolidityExecutionFramework(_ipcPath)
+{
+	ifstream file(_filename);
+	soltestAssert(file, "Cannot open test contract: \"" + _filename + "\".");
+	file.exceptions(ios::badbit);
+
+	m_source = parseSource(file);
+	parseExpectations(file);
+}
+
+bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+{
+	soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
+
+	bool success = true;
+	for (auto& test: m_tests)
+		test.reset();
+
+	for (auto& test: m_tests)
+	{
+		bytes output = callContractFunctionWithValueNoEncoding(
+			test.call.signature,
+			test.call.value,
+			test.call.arguments.rawBytes()
+		);
+
+		if ((m_transactionSuccessful == test.call.expectations.failure) || (output != test.call.expectations.rawBytes()))
+			success = false;
+
+		test.failure = !m_transactionSuccessful;
+		test.rawBytes = std::move(output);
+	}
+
+	if (!success)
+	{
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
+		for (auto const& test: m_tests)
+			_stream << formatFunctionCallTest(test, _linePrefix, false, true & _formatted);
+
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
+		for (auto const& test: m_tests)
+			_stream << formatFunctionCallTest(test, _linePrefix, true, true & _formatted);
+
+		AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix
+			<< "Attention: Updates on the test will apply the detected format displayed." << endl;
+		return false;
+	}
+	return true;
+}
+
+void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
+{
+	stringstream stream(m_source);
+	string line;
+	while (getline(stream, line))
+		_stream << _linePrefix << line << endl;
+}
+
+void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
+{
+	for (auto const& test: m_tests)
+		_stream << formatFunctionCallTest(test, "", true, false);
+}
+
+void SemanticTest::parseExpectations(istream& _stream)
+{
+	TestFileParser parser{_stream};
+	for (auto const& call: parser.parseFunctionCalls())
+		m_tests.emplace_back(FunctionCallTest{call, bytes{}, string{}});
+}
+
+bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments)
+{
+	auto output = compileAndRunWithoutCheck(m_source, _value, _contractName, _arguments);
+	return !output.empty() && m_transactionSuccessful;
+}

test/libsolidity/SemanticTest.h

@@ -0,0 +1,99 @@
+/*
+	This file is part of solidity.
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/util/TestFileParser.h>
+#include <test/libsolidity/SolidityExecutionFramework.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/TestCase.h>
+#include <liblangutil/Exceptions.h>
+#include <libdevcore/AnsiColorized.h>
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+/**
+ * Class that represents a semantic test (or end-to-end test) and allows running it as part of the
+ * boost unit test environment or isoltest. It reads the Solidity source and an additional comment
+ * section from the given file. This comment section should define a set of functions to be called
+ * and an expected result they return after being executed.
+ */
+class SemanticTest: public SolidityExecutionFramework, public TestCase
+{
+public:
+	/**
+	 * Represents a function call and the result it returned. It stores the call
+	 * representation itself, the actual byte result (if any) and a string representation
+	 * used for the interactive update routine provided by isoltest. It also provides
+	 * functionality to compare the actual result with the expectations attached to the
+	 * call object, as well as a way to reset the result if executed multiple times.
+	 */
+	struct FunctionCallTest
+	{
+		FunctionCall call;
+		bytes rawBytes;
+		std::string output;
+		bool failure = true;
+		/// Compares raw expectations (which are converted to a byte representation before),
+		/// and also the expected transaction status of the function call to the actual test results.
+		bool matchesExpectation() const
+		{
+			return failure == call.expectations.failure && rawBytes == call.expectations.rawBytes();
+		}
+		/// Resets current results in case the function was called and the result
+		/// stored already (e.g. if test case was updated via isoltest).
+		void reset()
+		{
+			failure = true;
+			rawBytes = bytes{};
+			output = std::string{};
+		}
+	};
+
+	static std::unique_ptr<TestCase> create(Config const& _options)
+	{ return std::make_unique<SemanticTest>(_options.filename, _options.ipcPath); }
+
+	explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath);
+
+	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
+	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;
+
+	/// Instantiates a test file parser that parses the additional comment section at the end of
+	/// the input stream \param _stream. Each function call is represented using a `FunctionCallTest`
+	/// and added to the list of call to be executed when `run()` is called.
+	/// Throws if parsing expectations failed.
+	void parseExpectations(std::istream& _stream);
+
+	/// Compiles and deploys currently held source.
+	/// Returns true if deployment was successful, false otherwise.
+	bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments);
+
+	std::string m_source;
+	std::vector<FunctionCallTest> m_tests;
+};
+
+}
+}
+}

test/libsolidity/SolidityExecutionFramework.cpp

@@ -28,7 +28,12 @@ using namespace dev::test;
 using namespace dev::solidity;
 using namespace dev::solidity::test;
 
-SolidityExecutionFramework::SolidityExecutionFramework() :
+SolidityExecutionFramework::SolidityExecutionFramework():
 	ExecutionFramework()
 {
 }
+
+SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath):
+	ExecutionFramework(_ipcPath)
+{
+}

test/libsolidity/SolidityExecutionFramework.h

@@ -43,6 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework
 
 public:
 	SolidityExecutionFramework();
+	SolidityExecutionFramework(std::string const& _ipcPath);
 
 	virtual bytes const& compileAndRunWithoutCheck(
 		std::string const& _sourceCode,

test/libsolidity/SyntaxTest.h

@@ -53,8 +53,8 @@ struct SyntaxTestError
 class SyntaxTest: AnalysisFramework, public TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
-	{ return std::unique_ptr<TestCase>(new SyntaxTest(_filename)); }
+	static std::unique_ptr<TestCase> create(Config const& _config)
+	{ return std::unique_ptr<TestCase>(new SyntaxTest(_config.filename)); }
 	SyntaxTest(std::string const& _filename);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;

test/libsolidity/semanticTests/smoke_test.sol

@@ -0,0 +1,17 @@
+contract C {
+    function f() public returns (uint) {
+        return 1;
+    }
+    function g(uint x, uint y) public returns (uint) {
+        return x - y;
+    }
+    function h() public payable returns (uint) {
+        return f();
+    }
+}
+// ----
+// f() -> 1
+// g(uint256,uint256): 1, -2 -> 3
+// h(), 1 ether -> 1
+// j() -> FAILURE
+// i() # Does not exist. # -> FAILURE # Reverts. #

test/libsolidity/semanticTests/smoke_test_multiline.sol

@@ -0,0 +1,11 @@
+contract C {
+    function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) {
+        return a + b + c + d + e;
+    }
+}
+// ----
+// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1
+// -> 5
+// g()
+// # g() does not exist #
+// -> FAILURE

test/libsolidity/semanticTests/smoke_test_multiline_comments.sol

@@ -0,0 +1,17 @@
+contract C {
+    function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) {
+        return a + b + c + d + e;
+    }
+}
+// ----
+// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1
+// # A comment on the function parameters. #
+// -> 5
+// f(uint256,uint256,uint256,uint256,uint256):
+// 1,
+// 1,
+// 1,
+// 1,
+// 1
+// -> 5
+// # Should return sum of all parameters. #

test/libsolidity/util/TestFileParser.cpp

@@ -85,6 +85,8 @@ vector<dev::solidity::test::FunctionCall> TestFileParser::parseFunctionCalls()
 
 				expect(Token::Arrow);
 				call.expectations = parseFunctionCallExpectations();
+
+				accept(Token::Newline, true);
 				call.expectations.comment = parseComment();
 
 				calls.emplace_back(std::move(call));
@@ -194,38 +196,45 @@ Parameter TestFileParser::parseParameter()
 	if (accept(Token::Newline, true))
 		parameter.format.newline = true;
 	auto literal = parseABITypeLiteral();
-	parameter.rawBytes = literal.first;
-	parameter.abiType = literal.second;
+	parameter.rawBytes = get<0>(literal);
+	parameter.abiType = get<1>(literal);
+	parameter.rawString = get<2>(literal);
 	return parameter;
 }
 
-pair<bytes, ABIType> TestFileParser::parseABITypeLiteral()
+tuple<bytes, ABIType, string> TestFileParser::parseABITypeLiteral()
 {
 	try
 	{
 		u256 number{0};
 		ABIType abiType{ABIType::None, 0};
+		string rawString;
 
 		if (accept(Token::Sub))
 		{
 			abiType = ABIType{ABIType::SignedDec, 32};
 			expect(Token::Sub);
-			number = convertNumber(parseNumber()) * -1;
+			rawString += formatToken(Token::Sub);
+			string parsed = parseNumber();
+			rawString += parsed;
+			number = convertNumber(parsed) * -1;
 		}
 		else
 		{
 			if (accept(Token::Number))
 			{
 				abiType = ABIType{ABIType::UnsignedDec, 32};
-				number = convertNumber(parseNumber());
+				string parsed = parseNumber();
+				rawString += parsed;
+				number = convertNumber(parsed);
 			}
 			else if (accept(Token::Failure, true))
 			{
 				abiType = ABIType{ABIType::Failure, 0};
-				return make_pair(bytes{}, abiType);
+				return make_tuple(bytes{}, abiType, rawString);
 			}
 		}
-		return make_pair(toBigEndian(number), abiType);
+		return make_tuple(toBigEndian(number), abiType, rawString);
 	}
 	catch (std::exception const&)
 	{

test/libsolidity/util/TestFileParser.h

@@ -114,7 +114,7 @@ struct ABIType
  */
 struct FormatInfo
 {
-	bool newline;
+	bool newline = false;
 };
 
 /**
@@ -132,6 +132,9 @@ struct Parameter
 	/// compared to the actual result of a function call
 	/// and used for validating it.
 	bytes rawBytes;
+	/// Stores the raw string representation of this parameter.
+	/// Used to print the unformatted arguments of a function call.
+	std::string rawString;
 	/// Types that were used to encode `rawBytes`. Expectations
 	/// are usually comma separated literals. Their type is auto-
 	/// detected and retained in order to format them later on.
@@ -327,13 +330,15 @@ private:
 	Parameter parseParameter();
 
 	/// Parses and converts the current literal to its byte representation and
-	/// preserves the chosen ABI type. Based on that type information, the driver of
-	/// this parser can format arguments, expectations and results. Supported types:
+	/// preserves the chosen ABI type, as well as a raw, unformatted string representation
+	/// of this literal.
+	/// Based on the type information retrieved, the driver of this parser may format arguments,
+	/// expectations and results. Supported types:
 	/// - unsigned and signed decimal number literals.
 	/// Returns invalid ABI type for empty literal. This is needed in order
 	/// to detect empty expectations. Throws a ParserError if data is encoded incorrectly or
 	/// if data type is not supported.
-	std::pair<bytes, ABIType> parseABITypeLiteral();
+	std::tuple<bytes, ABIType, std::string> parseABITypeLiteral();
 
 	/// Recursively parses an identifier or a tuple definition that contains identifiers
 	/// and / or parentheses like `((uint, uint), (uint, (uint, uint)), uint)`.

test/libsolidity/util/TestFileParserTests.cpp

@@ -56,7 +56,8 @@ void testFunctionCall(
 		bytes _expectations = bytes{},
 		u256 _value = 0,
 		string _argumentComment = "",
-		string _expectationComment = ""
+		string _expectationComment = "",
+		vector<string> _rawArguments = vector<string>{}
 )
 {
 	BOOST_REQUIRE_EQUAL(_call.expectations.failure, _failure);
@@ -67,6 +68,17 @@ void testFunctionCall(
 	BOOST_REQUIRE_EQUAL(_call.value, _value);
 	BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment);
 	BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment);
+
+	if (!_rawArguments.empty())
+	{
+		BOOST_REQUIRE_EQUAL(_call.arguments.parameters.size(), _rawArguments.size());
+		size_t index = 0;
+		for (Parameter const& param: _call.arguments.parameters)
+		{
+			BOOST_REQUIRE_EQUAL(param.rawString, _rawArguments[index]);
+			++index;
+		}
+	}
 }
 
 BOOST_AUTO_TEST_SUITE(TestFileParserTest)
@@ -112,11 +124,16 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success)
 {
 	char const* source = R"(
 		// f(uint256, uint256): 1, 1
+		// # Comment on the parameters. #
 		// ->
 		// # This call should not return a value, but still succeed. #
+		// f()
+		// # Comment on no parameters. #
+		// -> 1
+		// # This comment should be parsed. #
 	)";
 	auto const calls = parse(source);
-	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
 	testFunctionCall(
 		calls.at(0),
 		Mode::MultiLine,
@@ -125,9 +142,20 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success)
 		fmt::encodeArgs(1, 1),
 		fmt::encodeArgs(),
 		0,
-		"",
+		" Comment on the parameters. ",
 		" This call should not return a value, but still succeed. "
 	);
+	testFunctionCall(
+		calls.at(1),
+		Mode::MultiLine,
+		"f()",
+		false,
+		fmt::encodeArgs(),
+		fmt::encodeArgs(1),
+		0,
+		" Comment on no parameters. ",
+		" This comment should be parsed. "
+	);
 }
 
 BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success)
@@ -383,7 +411,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format)
 	);
 }
 
-BOOST_AUTO_TEST_CASE(call_signature)
+BOOST_AUTO_TEST_CASE(call_signature_valid)
 {
 	char const* source = R"(
 		// f(uint256, uint8, string) -> FAILURE
@@ -395,6 +423,27 @@ BOOST_AUTO_TEST_CASE(call_signature)
 	testFunctionCall(calls.at(1), Mode::SingleLine, "f(invalid,xyz,foo)", true);
 }
 
+BOOST_AUTO_TEST_CASE(call_raw_arguments)
+{
+	char const* source = R"(
+		// f(): 1, -2, -3 ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::SingleLine,
+		"f()",
+		false,
+		fmt::encodeArgs(1, -2, -3),
+		fmt::encodeArgs(),
+		0,
+		"",
+		"",
+		{"1", "-2", "-3"}
+	);
+}
+
 BOOST_AUTO_TEST_CASE(call_newline_invalid)
 {
 	char const* source = R"(

test/libyul/ObjectCompilerTest.h

@@ -40,9 +40,9 @@ namespace test
 class ObjectCompilerTest: public dev::solidity::test::TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new ObjectCompilerTest(_filename));
+		return std::unique_ptr<TestCase>(new ObjectCompilerTest(_config.filename));
 	}
 
 	explicit ObjectCompilerTest(std::string const& _filename);

test/libyul/YulOptimizerTest.h

@@ -41,9 +41,9 @@ namespace test
 class YulOptimizerTest: public dev::solidity::test::TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new YulOptimizerTest(_filename));
+		return std::unique_ptr<TestCase>(new YulOptimizerTest(_config.filename));
 	}
 
 	explicit YulOptimizerTest(std::string const& _filename);

test/tools/CMakeLists.txt

@@ -13,7 +13,9 @@ add_executable(isoltest
 	../Options.cpp
 	../Common.cpp
 	../TestCase.cpp
+	../libsolidity/util/TestFileParser.cpp
 	../libsolidity/SyntaxTest.cpp
+        ../libsolidity/SemanticTest.cpp
 	../libsolidity/AnalysisFramework.cpp
 	../libsolidity/SolidityExecutionFramework.cpp
 	../ExecutionFramework.cpp

test/tools/isoltest.cpp

@@ -64,8 +64,9 @@ public:
 		TestCase::TestCaseCreator _testCaseCreator,
 		string const& _name,
 		fs::path const& _path,
+		string const& _ipcPath,
 		bool _formatted
-	): m_testCaseCreator(_testCaseCreator), m_formatted(_formatted), m_name(_name), m_path(_path)
+	): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted)
 	{}
 
 	enum class Result
@@ -81,6 +82,7 @@ public:
 		TestCase::TestCaseCreator _testCaseCreator,
 		fs::path const& _basepath,
 		fs::path const& _path,
+		string const& _ipcPath,
 		bool const _formatted
 	);
 
@@ -96,9 +98,10 @@ private:
 	Request handleResponse(bool const _exception);
 
 	TestCase::TestCaseCreator m_testCaseCreator;
-	bool const m_formatted = false;
 	string const m_name;
 	fs::path const m_path;
+	string m_ipcPath;
+	bool const m_formatted = false;
 	unique_ptr<TestCase> m_test;
 	static bool m_exitRequested;
 };
@@ -115,25 +118,25 @@ TestTool::Result TestTool::process()
 
 	try
 	{
-		m_test = m_testCaseCreator(m_path.string());
+		m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath});
 		success = m_test->run(outputMessages, "  ", m_formatted);
 	}
 	catch(boost::exception const& _e)
 	{
 		AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
-			"Exception during syntax test: " << boost::diagnostic_information(_e) << endl;
+			"Exception during test: " << boost::diagnostic_information(_e) << endl;
 		return Result::Exception;
 	}
 	catch (std::exception const& _e)
 	{
 		AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
-			"Exception during syntax test: " << _e.what() << endl;
+			"Exception during test: " << _e.what() << endl;
 		return Result::Exception;
 	}
 	catch (...)
 	{
 		AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
-			"Unknown exception during syntax test." << endl;
+			"Unknown exception during test." << endl;
 		return Result::Exception;
 	}
 
@@ -199,6 +202,7 @@ TestStats TestTool::processPath(
 	TestCase::TestCaseCreator _testCaseCreator,
 	fs::path const& _basepath,
 	fs::path const& _path,
+	string const& _ipcPath,
 	bool const _formatted
 )
 {
@@ -230,7 +234,7 @@ TestStats TestTool::processPath(
 		else
 		{
 			++testCount;
-			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _formatted);
+			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted);
 			auto result = testTool.process();
 
 			switch(result)
@@ -291,6 +295,7 @@ boost::optional<TestStats> runTestSuite(
 	string const& _name,
 	fs::path const& _basePath,
 	fs::path const& _subdirectory,
+	string const& _ipcPath,
 	TestCase::TestCaseCreator _testCaseCreator,
 	bool _formatted
 )
@@ -303,7 +308,7 @@ boost::optional<TestStats> runTestSuite(
 		return {};
 	}
 
-	TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _formatted);
+	TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted);
 
 	cout << endl << _name << " Test Summary: ";
 	AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
@@ -327,11 +332,13 @@ int main(int argc, char *argv[])
 		TestTool::editor = "/usr/bin/editor";
 
 	fs::path testPath;
+	string ipcPath;
+	bool disableIPC = false;
 	bool disableSMT = false;
 	bool formatted = true;
 	po::options_description options(
 		R"(isoltest, tool for interactively managing test contracts.
-Usage: isoltest [Options] --testpath path
+Usage: isoltest [Options] --ipcpath ipcpath
 Interactively validates test contracts.
 
 Allowed options)",
@@ -340,6 +347,8 @@ Allowed options)",
 	options.add_options()
 		("help", "Show this help screen.")
 		("testpath", po::value<fs::path>(&testPath), "path to test files")
+		("ipcpath", po::value<string>(&ipcPath), "path to ipc socket")
+		("no-ipc", "disable semantic tests")
 		("no-smt", "disable SMT checker")
 		("no-color", "don't use colors")
 		("editor", po::value<string>(&TestTool::editor), "editor for opening contracts");
@@ -362,8 +371,23 @@ Allowed options)",
 
 		po::notify(arguments);
 
+		if (arguments.count("no-ipc"))
+			disableIPC = true;
+		else
+		{
+			solAssert(
+				!ipcPath.empty(),
+				"No ipc path specified. The --ipcpath argument is required, unless --no-ipc is used."
+			);
+			solAssert(
+				fs::exists(ipcPath),
+				"Invalid ipc path specified."
+			);
+		}
+
 		if (arguments.count("no-smt"))
 			disableSMT = true;
+
 	}
 	catch (std::exception const& _exception)
 	{
@@ -380,10 +404,13 @@ Allowed options)",
 	// Interactive tests are added in InteractiveTests.h
 	for (auto const& ts: g_interactiveTestsuites)
 	{
+		if (ts.ipc && disableIPC)
+			continue;
+
 		if (ts.smt && disableSMT)
 			continue;
 
-		if (auto stats = runTestSuite(ts.title, testPath / ts.path, ts.subpath, ts.testCaseCreator, formatted))
+		if (auto stats = runTestSuite(ts.title, testPath / ts.path, ts.subpath, ipcPath, ts.testCaseCreator, formatted))
 			global_stats += *stats;
 		else
 			return 1;