Adds semantic tests to test framework and isoltest.

test/libsolidity/SemanticTest.cpp

@@ -28,30 +28,33 @@
 using namespace dev;
 using namespace solidity;
 using namespace dev::solidity::test;
-using namespace dev::solidity::test::formatting;
+using namespace dev::formatting;
 using namespace std;
-namespace fs = boost::filesystem;
 using namespace boost;
 using namespace boost::algorithm;
 using namespace boost::unit_test;
+namespace fs = boost::filesystem;
 
 namespace
 {
-	using ParamList = dev::solidity::test::ParameterList;
-	using FunctionCallTest = dev::solidity::test::SemanticTest::FunctionCallTest;
+	using FunctionCallTest = SemanticTest::FunctionCallTest;
 	using FunctionCall = dev::solidity::test::FunctionCall;
+	using ParamList = dev::solidity::test::ParameterList;
 
-	string formatBytes(bytes const& _bytes, ParamList const& _params, bool const _formatInvalid = false)
+
+	string formatBytes(bytes const& _bytes, ParamList const& _params)
 	{
 		stringstream resultStream;
 		if (_bytes.empty())
-			resultStream.str();
+			return {};
 		auto it = _bytes.begin();
 		for (auto const& param: _params)
 		{
-			bytes byteRange{it, it + param.abiType.size};
-			// FIXME Check range
-			// TODO Check range
+			long offset = static_cast<long>(param.abiType.size);
+			auto offsetIter = it + offset;
+			soltestAssert(offsetIter <= _bytes.end(), "Byte range can not be extended past the end of given bytes.");
+
+			bytes byteRange{it, offsetIter};
 			switch (param.abiType.type)
 			{
 			case ABIType::SignedDec:
@@ -71,22 +74,29 @@ namespace
 					resultStream << fromBigEndian<u256>(byteRange);
 				break;
 			case ABIType::Failure:
-				// If expectations are empty, the encoding type is invalid.
-				// In order to still print the actual result even if
-				// empty expectations were detected, it must be forced.
-				if (_formatInvalid)
-					resultStream << fromBigEndian<u256>(byteRange);
 				break;
 			case ABIType::None:
-				// If expectations are empty, the encoding type is NONE.
-				if (_formatInvalid)
-					resultStream << fromBigEndian<u256>(byteRange);
 				break;
 			}
-			it += param.abiType.size;
+			it += offset;
 			if (it != _bytes.end() && !(param.abiType.type == ABIType::None))
 				resultStream << ", ";
 		}
+		soltestAssert(it == _bytes.end(), "Parameter encoding too short for the given byte range.");
+		return resultStream.str();
+	}
+
+	string formatRawArguments(ParamList const& _params, string const& _linePrefix = "")
+	{
+		stringstream resultStream;
+		for (auto const& param: _params)
+		{
+			if (param.format.newline)
+				resultStream << endl << _linePrefix << "//";
+			resultStream << " " << param.rawString;
+			if (&param != &_params.back())
+				resultStream << ",";
+		}
 		return resultStream.str();
 	}
 
@@ -94,41 +104,86 @@ namespace
 		FunctionCallTest const& _test,
 		string const& _linePrefix = "",
 		bool const _renderResult = false,
-		bool const _higlight = false
+		bool const _highlight = false
 	)
 	{
+		using namespace soltest;
+		using Token = soltest::Token;
+
 		stringstream _stream;
 		FunctionCall call = _test.call;
-		bool hightlight = !_test.matchesExpectation() && _higlight;
+		bool highlight = !_test.matchesExpectation() && _highlight;
 
 		auto formatOutput = [&](bool const _singleLine)
 		{
-			_stream << _linePrefix << "// " << call.signature;
+			string ws = " ";
+			string arrow = formatToken(Token::Arrow);
+			string colon = formatToken(Token::Colon);
+			string comma = formatToken(Token::Comma);
+			string comment = formatToken(Token::Comment);
+			string ether = formatToken(Token::Ether);
+			string newline = formatToken(Token::Newline);
+			string failure = formatToken(Token::Failure);
+
+			/// Prints the function signature. This is the same independent from the display-mode.
+			_stream << _linePrefix << newline << ws << call.signature;
 			if (call.value > u256(0))
-				_stream << TestFileParser::formatToken(SoltToken::Comma)
-						<< call.value << " "
-						<< TestFileParser::formatToken(SoltToken::Ether);
+				_stream << comma << ws << call.value << ws << ether;
 			if (!call.arguments.rawBytes().empty())
-				_stream << ": "
-						<< formatBytes(call.arguments.rawBytes(), call.arguments.parameters);
-			if (!_singleLine)
-				_stream << endl << _linePrefix << "// ";
+			{
+				string output = formatRawArguments(call.arguments.parameters, _linePrefix);
+				_stream << colon << output;
+			}
+
+			/// Prints comments on the function parameters and the arrow taking
+			/// the display-mode into account.
 			if (_singleLine)
-				_stream << " ";
-			_stream << "-> ";
-			if (!_singleLine)
-				_stream << endl << _linePrefix << "// ";
-			if (hightlight)
-				_stream << formatting::RED_BACKGROUND;
-			bytes output;
-			if (_renderResult)
-				output = call.expectations.rawBytes();
+			{
+				if (!call.arguments.comment.empty())
+					_stream << ws << comment << call.arguments.comment << comment;
+				_stream << ws << arrow << ws;
+			}
 			else
-				output = _test.rawBytes;
-			if (!output.empty())
-				_stream << formatBytes(output, call.expectations.result);
-			if (hightlight)
-				_stream << formatting::RESET;
+			{
+				_stream << endl << _linePrefix << newline << ws;
+				if (!call.arguments.comment.empty())
+				{
+					 _stream << comment << call.arguments.comment << comment;
+					 _stream << endl << _linePrefix << newline << ws;
+				}
+				_stream << arrow << ws;
+			}
+
+			/// Print either the expected output or the actual result output
+			string result;
+			if (!_renderResult)
+			{
+				bytes output = call.expectations.rawBytes();
+				bool const isFailure = call.expectations.failure;
+				result = isFailure ? failure : formatBytes(output, call.expectations.result);
+			}
+			else
+			{
+				bytes output = _test.rawBytes;
+				bool const isFailure = _test.failure;
+				result = isFailure ? failure : formatBytes(output, call.expectations.result);
+			}
+			AnsiColorized(_stream, highlight, {RED_BACKGROUND}) << result;
+
+			/// Print comments on expectations taking the display-mode into account.
+			if (_singleLine)
+			{
+				if (!call.expectations.comment.empty())
+					_stream << ws << comment << call.expectations.comment << comment;
+			}
+			else
+			{
+				if (!call.expectations.comment.empty())
+				{
+					_stream << endl << _linePrefix << newline << ws;
+					_stream << comment << call.expectations.comment << comment;
+				}
+			}
 		};
 
 		if (call.displayMode == FunctionCall::DisplayMode::SingleLine)
@@ -145,8 +200,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
 	SolidityExecutionFramework(_ipcPath)
 {
 	ifstream file(_filename);
-	if (!file)
-		BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\"."));
+	soltestAssert(file, "Cannot open test contract: \"" + _filename + "\".");
 	file.exceptions(ios::badbit);
 
 	m_source = parseSource(file);
@@ -155,8 +209,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
 
 bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
-	if (!deploy("", 0, bytes()))
-		BOOST_THROW_EXCEPTION(runtime_error("Failed to deploy contract."));
+	soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
 
 	bool success = true;
 	for (auto& test: m_tests)
@@ -179,15 +232,15 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _
 
 	if (!success)
 	{
-		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
 		for (auto const& test: m_tests)
-			_stream << formatFunctionCallTest(test, _linePrefix, false, true);
+			_stream << formatFunctionCallTest(test, _linePrefix, false, true & _formatted);
 
-		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
+		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
 		for (auto const& test: m_tests)
-			_stream << formatFunctionCallTest(test, _linePrefix, true, true);
+			_stream << formatFunctionCallTest(test, _linePrefix, true, true & _formatted);
 
-		FormattedScope(_stream, _formatted, {BOLD, RED}) << _linePrefix
+		AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix
 			<< "Attention: Updates on the test will apply the detected format displayed." << endl;
 		return false;
 	}
@@ -202,10 +255,10 @@ void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool
 		_stream << _linePrefix << line << endl;
 }
 
-void SemanticTest::printUpdatedExpectations(ostream& _stream, string const& _linePrefix) const
+void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
 {
 	for (auto const& test: m_tests)
-		_stream << formatFunctionCallTest(test, _linePrefix, false, false);
+		_stream << formatFunctionCallTest(test, "", true, false);
 }
 
 void SemanticTest::parseExpectations(istream& _stream)

test/libsolidity/SemanticTest.h

@@ -15,11 +15,11 @@
 #pragma once
 
 #include <test/libsolidity/util/TestFileParser.h>
-#include <test/libsolidity/FormattedScope.h>
 #include <test/libsolidity/SolidityExecutionFramework.h>
 #include <test/libsolidity/AnalysisFramework.h>
 #include <test/TestCase.h>
 #include <liblangutil/Exceptions.h>
+#include <libdevcore/AnsiColorized.h>
 
 #include <iosfwd>
 #include <string>

test/libsolidity/semanticTests/smoke_test.sol

@@ -0,0 +1,17 @@
+contract C {
+    function f() public returns (uint) {
+        return 1;
+    }
+    function g(uint x, uint y) public returns (uint) {
+        return x - y;
+    }
+    function h() public payable returns (uint) {
+        return f();
+    }
+}
+// ----
+// f() -> 1
+// g(uint256,uint256): 1, -2 -> 3
+// h(), 1 ether -> 1
+// j() -> FAILURE
+// i() # Does not exist. # -> FAILURE # Reverts. #

test/libsolidity/semanticTests/smoke_test_multiline.sol

@@ -0,0 +1,11 @@
+contract C {
+    function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) {
+        return a + b + c + d + e;
+    }
+}
+// ----
+// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1
+// -> 5
+// g()
+// # g() does not exist #
+// -> FAILURE

test/libsolidity/semanticTests/smoke_test_multiline_comments.sol

@@ -0,0 +1,17 @@
+contract C {
+    function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) {
+        return a + b + c + d + e;
+    }
+}
+// ----
+// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1
+// # A comment on the function parameters. #
+// -> 5
+// f(uint256,uint256,uint256,uint256,uint256):
+// 1,
+// 1,
+// 1,
+// 1,
+// 1
+// -> 5
+// # Should return sum of all parameters. #

test/libsolidity/util/TestFileParser.cpp

@@ -85,6 +85,8 @@ vector<dev::solidity::test::FunctionCall> TestFileParser::parseFunctionCalls()
 
 				expect(Token::Arrow);
 				call.expectations = parseFunctionCallExpectations();
+
+				accept(Token::Newline, true);
 				call.expectations.comment = parseComment();
 
 				calls.emplace_back(std::move(call));
@@ -194,38 +196,45 @@ Parameter TestFileParser::parseParameter()
 	if (accept(Token::Newline, true))
 		parameter.format.newline = true;
 	auto literal = parseABITypeLiteral();
-	parameter.rawBytes = literal.first;
-	parameter.abiType = literal.second;
+	parameter.rawBytes = get<0>(literal);
+	parameter.abiType = get<1>(literal);
+	parameter.rawString = get<2>(literal);
 	return parameter;
 }
 
-pair<bytes, ABIType> TestFileParser::parseABITypeLiteral()
+tuple<bytes, ABIType, string> TestFileParser::parseABITypeLiteral()
 {
 	try
 	{
 		u256 number{0};
 		ABIType abiType{ABIType::None, 0};
+		string rawString;
 
 		if (accept(Token::Sub))
 		{
 			abiType = ABIType{ABIType::SignedDec, 32};
 			expect(Token::Sub);
-			number = convertNumber(parseNumber()) * -1;
+			rawString += formatToken(Token::Sub);
+			string parsed = parseNumber();
+			rawString += parsed;
+			number = convertNumber(parsed) * -1;
 		}
 		else
 		{
 			if (accept(Token::Number))
 			{
 				abiType = ABIType{ABIType::UnsignedDec, 32};
-				number = convertNumber(parseNumber());
+				string parsed = parseNumber();
+				rawString += parsed;
+				number = convertNumber(parsed);
 			}
 			else if (accept(Token::Failure, true))
 			{
 				abiType = ABIType{ABIType::Failure, 0};
-				return make_pair(bytes{}, abiType);
+				return make_tuple(bytes{}, abiType, rawString);
 			}
 		}
-		return make_pair(toBigEndian(number), abiType);
+		return make_tuple(toBigEndian(number), abiType, rawString);
 	}
 	catch (std::exception const&)
 	{

test/libsolidity/util/TestFileParser.h

@@ -114,7 +114,7 @@ struct ABIType
  */
 struct FormatInfo
 {
-	bool newline;
+	bool newline = false;
 };
 
 /**
@@ -132,6 +132,9 @@ struct Parameter
 	/// compared to the actual result of a function call
 	/// and used for validating it.
 	bytes rawBytes;
+	/// Stores the raw string representation of this parameter.
+	/// Used to print the unformatted arguments of a function call.
+	std::string rawString;
 	/// Types that were used to encode `rawBytes`. Expectations
 	/// are usually comma separated literals. Their type is auto-
 	/// detected and retained in order to format them later on.
@@ -327,13 +330,15 @@ private:
 	Parameter parseParameter();
 
 	/// Parses and converts the current literal to its byte representation and
-	/// preserves the chosen ABI type. Based on that type information, the driver of
-	/// this parser can format arguments, expectations and results. Supported types:
+	/// preserves the chosen ABI type, as well as a raw, unformatted string representation
+	/// of this literal.
+	/// Based on the type information retrieved, the driver of this parser may format arguments,
+	/// expectations and results. Supported types:
 	/// - unsigned and signed decimal number literals.
 	/// Returns invalid ABI type for empty literal. This is needed in order
 	/// to detect empty expectations. Throws a ParserError if data is encoded incorrectly or
 	/// if data type is not supported.
-	std::pair<bytes, ABIType> parseABITypeLiteral();
+	std::tuple<bytes, ABIType, std::string> parseABITypeLiteral();
 
 	/// Recursively parses an identifier or a tuple definition that contains identifiers
 	/// and / or parentheses like `((uint, uint), (uint, (uint, uint)), uint)`.

test/libsolidity/util/TestFileParserTests.cpp

@@ -56,7 +56,8 @@ void testFunctionCall(
 		bytes _expectations = bytes{},
 		u256 _value = 0,
 		string _argumentComment = "",
-		string _expectationComment = ""
+		string _expectationComment = "",
+		vector<string> _rawArguments = vector<string>{}
 )
 {
 	BOOST_REQUIRE_EQUAL(_call.expectations.failure, _failure);
@@ -67,6 +68,17 @@ void testFunctionCall(
 	BOOST_REQUIRE_EQUAL(_call.value, _value);
 	BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment);
 	BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment);
+
+	if (!_rawArguments.empty())
+	{
+		BOOST_REQUIRE_EQUAL(_call.arguments.parameters.size(), _rawArguments.size());
+		size_t index = 0;
+		for (Parameter const& param: _call.arguments.parameters)
+		{
+			BOOST_REQUIRE_EQUAL(param.rawString, _rawArguments[index]);
+			++index;
+		}
+	}
 }
 
 BOOST_AUTO_TEST_SUITE(TestFileParserTest)
@@ -112,11 +124,16 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success)
 {
 	char const* source = R"(
 		// f(uint256, uint256): 1, 1
+		// # Comment on the parameters. #
 		// ->
 		// # This call should not return a value, but still succeed. #
+		// f()
+		// # Comment on no parameters. #
+		// -> 1
+		// # This comment should be parsed. #
 	)";
 	auto const calls = parse(source);
-	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls.size(), 2);
 	testFunctionCall(
 		calls.at(0),
 		Mode::MultiLine,
@@ -125,9 +142,20 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success)
 		fmt::encodeArgs(1, 1),
 		fmt::encodeArgs(),
 		0,
-		"",
+		" Comment on the parameters. ",
 		" This call should not return a value, but still succeed. "
 	);
+	testFunctionCall(
+		calls.at(1),
+		Mode::MultiLine,
+		"f()",
+		false,
+		fmt::encodeArgs(),
+		fmt::encodeArgs(1),
+		0,
+		" Comment on no parameters. ",
+		" This comment should be parsed. "
+	);
 }
 
 BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success)
@@ -383,7 +411,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format)
 	);
 }
 
-BOOST_AUTO_TEST_CASE(call_signature)
+BOOST_AUTO_TEST_CASE(call_signature_valid)
 {
 	char const* source = R"(
 		// f(uint256, uint8, string) -> FAILURE
@@ -395,6 +423,27 @@ BOOST_AUTO_TEST_CASE(call_signature)
 	testFunctionCall(calls.at(1), Mode::SingleLine, "f(invalid,xyz,foo)", true);
 }
 
+BOOST_AUTO_TEST_CASE(call_raw_arguments)
+{
+	char const* source = R"(
+		// f(): 1, -2, -3 ->
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::SingleLine,
+		"f()",
+		false,
+		fmt::encodeArgs(1, -2, -3),
+		fmt::encodeArgs(),
+		0,
+		"",
+		"",
+		{"1", "-2", "-3"}
+	);
+}
+
 BOOST_AUTO_TEST_CASE(call_newline_invalid)
 {
 	char const* source = R"(