[isoltest] Add support for call side-effects.

test/libsolidity/SemanticTest.cpp

@@ -58,6 +58,7 @@ SemanticTest::SemanticTest(
 	m_sources(m_reader.sources()),
 	m_lineOffset(m_reader.lineNumber()),
 	m_builtins(makeBuiltins()),
+	m_sideEffectHooks(makeSideEffectHooks()),
 	m_enforceViaYul(_enforceViaYul),
 	m_enforceCompileToEwasm(_enforceCompileToEwasm),
 	m_enforceGasCost(_enforceGasCost),
@@ -127,12 +128,22 @@ map<string, Builtin> SemanticTest::makeBuiltins()
 {
 	return {
 		{
-			"smokeTest",
+			"isoltest_builtin_test",
 			[](FunctionCall const&) -> optional<bytes>
 			{
 				return util::toBigEndian(u256(0x1234));
 			}
 		},
+		{
+			"isoltest_side_effects_test",
+			[](FunctionCall const& _call) -> optional<bytes>
+			{
+				if (_call.arguments.parameters.empty())
+					return util::toBigEndian(0);
+				else
+					return _call.arguments.rawBytes();
+			}
+		},
 		{
 			"balance",
 			[this](FunctionCall const& _call) -> optional<bytes>
@@ -167,6 +178,23 @@ map<string, Builtin> SemanticTest::makeBuiltins()
 	};
 }
 
+
+vector<SideEffectHook> SemanticTest::makeSideEffectHooks() const
+{
+	return {
+		[](FunctionCall const& _call) -> vector<string>
+		{
+			if (_call.signature == "isoltest_side_effects_test")
+			{
+				vector<string> result;
+				for (auto const& argument: _call.arguments.parameters)
+					result.emplace_back(toHex(argument.rawBytes));
+				return result;
+			}
+			return {};
+		}};
+}
+
 TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	TestResult result = TestResult::Success;
@@ -322,6 +350,13 @@ TestCase::TestResult SemanticTest::runTest(
 			test.setRawBytes(move(output));
 			test.setContractABI(m_compiler.contractABI(m_compiler.lastContractName(m_sources.mainSourceFile)));
 		}
+
+		vector<string> effects;
+		for (SideEffectHook const& hook: m_sideEffectHooks)
+			effects += hook(test.call());
+		test.setSideEffects(move(effects));
+
+		success &= test.call().expectedSideEffects == test.call().actualSideEffects;
 	}
 
 	if (!m_testCaseWantsYulRun && _isYulRun)
@@ -538,8 +573,7 @@ void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePre
 
 void SemanticTest::parseExpectations(istream& _stream)
 {
-	TestFileParser parser{_stream, m_builtins};
+	m_tests += TestFileParser{_stream, m_builtins}.parseFunctionCalls(m_lineOffset);
-	m_tests += parser.parseFunctionCalls(m_lineOffset);
 }
 
 bool SemanticTest::deploy(

test/libsolidity/SemanticTest.h

@@ -81,10 +81,12 @@ private:
 	TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _isYulRun, bool _isEwasmRun);
 	bool checkGasCostExpectation(TestFunctionCall& io_test, bool _compileViaYul) const;
 	std::map<std::string, Builtin> makeBuiltins();
+	std::vector<SideEffectHook> makeSideEffectHooks() const;
 	SourceMap m_sources;
 	std::size_t m_lineOffset;
 	std::vector<TestFunctionCall> m_tests;
 	std::map<std::string, Builtin> const m_builtins;
+	std::vector<SideEffectHook> const m_sideEffectHooks;
 	bool m_testCaseWantsYulRun = false;
 	bool m_testCaseWantsEwasmRun = false;
 	bool m_testCaseWantsLegacyRun = true;

test/libsolidity/semanticTests/isoltestTesting/builtins.sol

@@ -3,5 +3,4 @@ contract SmokeTest {
 // ====
 // compileViaYul: also
 // ----
-// constructor()
+// isoltest_builtin_test -> 0x1234
-// smokeTest -> 0x1234

test/libsolidity/semanticTests/isoltestTesting/effects.sol

@@ -0,0 +1,15 @@
+contract SmokeTest {
+}
+// ====
+// compileViaYul: also
+// ----
+// isoltest_side_effects_test -> 0
+// isoltest_side_effects_test: 0x1234 -> 0x1234
+// ~ 0000000000000000000000000000000000000000000000000000000000001234
+// isoltest_side_effects_test: 0x1234, 0x2345 # comment # -> 0x1234, 0x2345
+// ~ 0000000000000000000000000000000000000000000000000000000000001234
+// ~ 0000000000000000000000000000000000000000000000000000000000002345
+// isoltest_side_effects_test: 0x1234, 0x2345, 0x3456 -> 0x1234, 0x2345, 0x3456 # comment #
+// ~ 0000000000000000000000000000000000000000000000000000000000001234
+// ~ 0000000000000000000000000000000000000000000000000000000000002345
+// ~ 0000000000000000000000000000000000000000000000000000000000003456

test/libsolidity/util/SoltestTypes.h

@@ -38,6 +38,7 @@ namespace solidity::frontend::test
 	T(LBrace, "{", 0)                  \
 	T(RBrace, "}", 0)                  \
 	T(Sub,    "-", 0)                  \
+	T(Tilde,  "~", 0)                  \
 	T(Colon,  ":", 0)                  \
 	T(Comma,  ",", 0)                  \
 	T(Period, ".", 0)                  \
@@ -302,8 +303,13 @@ struct FunctionCall
 	/// Marks this function call as "short-handed", meaning
 	/// no `->` declared.
 	bool omitsArrow = true;
+	/// A textual representation of the expected side-effect of the function call.
+	std::vector<std::string> expectedSideEffects{};
+	/// A textual representation of the actual side-effect of the function call.
+	std::vector<std::string> actualSideEffects{};
 };
 
 using Builtin = std::function<std::optional<bytes>(FunctionCall const&)>;
+using SideEffectHook = std::function<std::vector<std::string>(FunctionCall const&)>;
 
 }

test/libsolidity/util/TestFileParser.cpp

@@ -42,7 +42,7 @@ using Token = soltest::Token;
 
 char TestFileParser::Scanner::peek() const noexcept
 {
-	if (std::distance(m_char, m_line.end()) < 2)
+	if (std::distance(m_char, m_source.end()) < 2)
 		return '\0';
 
 	auto next = m_char;
@@ -97,7 +97,6 @@ vector<solidity::frontend::test::FunctionCall> TestFileParser::parseFunctionCall
 					else
 					{
 						FunctionCall call;
-
 						if (accept(Token::Library, true))
 						{
 							expect(Token::Colon);
@@ -154,7 +153,10 @@ vector<solidity::frontend::test::FunctionCall> TestFileParser::parseFunctionCall
 								call.kind = FunctionCall::Kind::Constructor;
 						}
 
-						calls.emplace_back(std::move(call));
+						accept(Token::Newline, true);
+						call.expectedSideEffects = parseFunctionCallSideEffects();
+
+						calls.emplace_back(move(call));
 					}
 				}
 				catch (TestParserError const& _e)
@@ -169,6 +171,22 @@ vector<solidity::frontend::test::FunctionCall> TestFileParser::parseFunctionCall
 	return calls;
 }
 
+vector<string> TestFileParser::parseFunctionCallSideEffects()
+{
+	vector<string> result;
+	while (accept(Token::Tilde, false))
+	{
+		string effect = m_scanner.currentLiteral();
+		result.emplace_back(effect);
+		soltestAssert(m_scanner.currentToken() == Token::Tilde, "");
+		m_scanner.scanNextToken();
+		if (m_scanner.currentToken() == Token::Newline)
+			m_scanner.scanNextToken();
+	}
+
+	return result;
+}
+
 bool TestFileParser::accept(Token _token, bool const _expect)
 {
 	if (m_scanner.currentToken() != _token)
@@ -492,9 +510,10 @@ string TestFileParser::parseString()
 void TestFileParser::Scanner::readStream(istream& _stream)
 {
 	std::string line;
+	// TODO: std::getline(..) removes newlines '\n', if present. This could be improved.
 	while (std::getline(_stream, line))
-		m_line += line;
+		m_source += line;
-	m_char = m_line.begin();
+	m_char = m_source.begin();
 }
 
 void TestFileParser::Scanner::scanNextToken()
@@ -545,6 +564,10 @@ void TestFileParser::Scanner::scanNextToken()
 			else
 				selectToken(Token::Sub);
 			break;
+		case '~':
+			advance();
+			selectToken(Token::Tilde, readLine());
+			break;
 		case ':':
 			selectToken(Token::Colon);
 			break;
@@ -588,7 +611,7 @@ void TestFileParser::Scanner::scanNextToken()
 			}
 			else if (langutil::isWhiteSpace(current()))
 				selectToken(Token::Whitespace);
-			else if (isEndOfLine())
+			else if (isEndOfFile())
 			{
 				m_currentToken = Token::EOS;
 				m_currentLiteral = "";
@@ -601,6 +624,21 @@ void TestFileParser::Scanner::scanNextToken()
 	while (m_currentToken == Token::Whitespace);
 }
 
+string TestFileParser::Scanner::readLine()
+{
+	string line;
+	// Right now the scanner discards all (real) new-lines '\n' in TestFileParser::Scanner::readStream(..).
+	// Token::NewLine is defined as `//`, and NOT '\n'. We are just searching here for the next `/`.
+	// Note that `/` anywhere else than at the beginning of a line is currently forbidden (TODO: until we fix newline handling).
+	// Once the end of the file would be reached (or beyond), peek() will return '\0'.
+	while (peek() != '\0' && peek() != '/')
+	{
+		advance();
+		line += current();
+	}
+	return line;
+}
+
 string TestFileParser::Scanner::scanComment()
 {
 	string comment;

test/libsolidity/util/TestFileParser.h

@@ -90,20 +90,21 @@ private:
 		std::string scanDecimalNumber();
 		std::string scanHexNumber();
 		std::string scanString();
+		std::string readLine();
 		char scanHexPart();
 
 	private:
 		/// Advances current position in the input stream.
 		void advance(unsigned n = 1)
 		{
-			solAssert(m_char != m_line.end(), "Cannot advance beyond end.");
+			solAssert(m_char != m_source.end(), "Cannot advance beyond end.");
 			m_char = std::next(m_char, n);
 		}
 
 		/// Returns the current character or '\0' if at end of input.
 		char current() const noexcept
 		{
-			if (m_char == m_line.end())
+			if (m_char == m_source.end())
 				return '\0';
 
 			return *m_char;
@@ -113,10 +114,10 @@ private:
 		/// without advancing the input stream iterator.
 		char peek() const noexcept;
 
-		/// Returns true if the end of a line is reached, false otherwise.
+		/// Returns true if the end of the file is reached, false otherwise.
-		bool isEndOfLine() const { return m_char == m_line.end(); }
+		bool isEndOfFile() const { return m_char == m_source.end(); }
 
-		std::string m_line;
+		std::string m_source;
 		std::string::const_iterator m_char;
 
 		std::string m_currentLiteral;
@@ -180,6 +181,9 @@ private:
 	/// Parses the current string literal.
 	std::string parseString();
 
+	/// Parses the expected side effects of a function call execution.
+	std::vector<std::string> parseFunctionCallSideEffects();
+
 	/// Checks whether a builtin function with the given signature exist.
 	/// @returns true, if builtin found, false otherwise
 	bool isBuiltinFunction(std::string const& _signature);

test/libsolidity/util/TestFileParserTests.cpp

@@ -23,6 +23,7 @@
 #include <string>
 #include <tuple>
 #include <boost/test/unit_test.hpp>
+#include <boost/algorithm/string/trim.hpp>
 #include <liblangutil/Exceptions.h>
 #include <test/ExecutionFramework.h>
 
@@ -42,12 +43,10 @@ using Mode = FunctionCall::DisplayMode;
 namespace
 {
 
-vector<FunctionCall> parse(string const& _source)
+vector<FunctionCall> parse(string const& _source, std::map<std::string, Builtin> const& _builtins = {})
 {
-	static std::map<std::string, Builtin> const builtins = {};
-
 	istringstream stream{_source, ios_base::out};
-	return TestFileParser{stream, builtins}.parseFunctionCalls(0);
+	return TestFileParser{stream, _builtins}.parseFunctionCalls(0);
 }
 
 void testFunctionCall(
@@ -100,7 +99,7 @@ BOOST_AUTO_TEST_CASE(smoke_test)
 	BOOST_REQUIRE_EQUAL(parse(source).size(), 0);
 }
 
-BOOST_AUTO_TEST_CASE(call_success)
+BOOST_AUTO_TEST_CASE(call_succees)
 {
 	char const* source = R"(
 		// success() ->
@@ -959,6 +958,112 @@ BOOST_AUTO_TEST_CASE(library)
 	);
 }
 
+BOOST_AUTO_TEST_CASE(call_effects)
+{
+	std::map<std::string, Builtin> builtins;
+	builtins["builtin_returning_call_effect"] = [](FunctionCall const&) -> std::optional<bytes>
+	{
+		return util::toBigEndian(u256(0x1234));
+	};
+	builtins["builtin_returning_call_effect_no_ret"] = [](FunctionCall const&) -> std::optional<bytes>
+	{
+		return {};
+	};
+
+	char const* source = R"(
+		// builtin_returning_call_effect -> 1
+		// ~ bla
+		// ~ bla bla
+		// ~ bla bla bla
+	)";
+	vector<FunctionCall> calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 3);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "bla");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[1]), "bla bla");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[2]), "bla bla bla");
+	source = R"(
+		// builtin_returning_call_effect -> 1
+		// ~ bla
+		// ~ bla bla
+		// builtin_returning_call_effect -> 2
+		// ~ bla bla bla
+		// builtin_returning_call_effect -> 3
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 3);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 2);
+	BOOST_REQUIRE_EQUAL(calls[1].expectedSideEffects.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[2].expectedSideEffects.size(), 0);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "bla");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[1]), "bla bla");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[1].expectedSideEffects[0]), "bla bla bla");
+	source = R"(
+		// builtin_returning_call_effect -> 1
+		// ~ bla
+		// ~ bla bla bla
+		// ~ abc ~ def ~ ghi
+		// ~ ~ ~
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 4);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "bla");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[1]), "bla bla bla");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[2]), "abc ~ def ~ ghi");
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[3]), "~ ~");
+	source = R"(
+		// builtin_returning_call_effect_no_ret ->
+		// ~ hello world
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 1);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "hello world");
+	source = R"(
+		// builtin_returning_call_effect -> 1
+		// ~
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 1);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "");
+	source = R"(
+		// builtin_returning_call_effect -> 1 # a comment #
+		// ~ hello world
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 1);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "hello world");
+	source = R"(
+		// builtin_returning_call_effect_no_ret -> # another comment #
+		// ~ hello world
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 1);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "hello world");
+	source = R"(
+		// builtin_returning_call_effect_no_ret # another comment #
+		// ~ hello world
+	)";
+	calls = parse(source, builtins);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	BOOST_REQUIRE_EQUAL(calls[0].expectedSideEffects.size(), 1);
+	BOOST_REQUIRE_EQUAL(boost::trim_copy(calls[0].expectedSideEffects[0]), "hello world");
+	source = R"(
+		// builtin_returning_call_effect_no_ret # another comment #
+		// ~ hello/world
+	)";
+	BOOST_CHECK_THROW(parse(source, builtins), std::exception);
+	source = R"(
+		// builtin_returning_call_effect_no_ret # another comment #
+		// ~ hello//world
+	)";
+	BOOST_CHECK_THROW(parse(source, builtins), std::exception);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 }

test/libsolidity/util/TestFunctionCall.cpp

@@ -197,6 +197,23 @@ string TestFunctionCall::format(
 		}
 
 		stream << formatGasExpectations(_linePrefix, _renderMode == RenderMode::ExpectedValuesActualGas, _interactivePrint);
+
+		vector<string> sideEffects;
+		if (_renderMode == RenderMode::ExpectedValuesExpectedGas || _renderMode == RenderMode::ExpectedValuesActualGas)
+			sideEffects = m_call.expectedSideEffects;
+		else
+			sideEffects = m_call.actualSideEffects;
+
+		if (!sideEffects.empty())
+		{
+			stream << std::endl;
+			for (string const& effect: sideEffects)
+			{
+				stream << _linePrefix << "// ~ " << effect;
+				if (effect != *sideEffects.rbegin())
+					stream << std::endl;
+			}
+		}
 	};
 
 	formatOutput(m_call.displayMode == FunctionCall::DisplayMode::SingleLine);

test/libsolidity/util/TestFunctionCall.h

@@ -93,6 +93,7 @@ public:
 	void setRawBytes(const bytes _rawBytes) { m_rawBytes = _rawBytes; }
 	void setGasCost(std::string const& _runType, u256 const& _gasCost) { m_gasCosts[_runType] = _gasCost; }
 	void setContractABI(Json::Value _contractABI) { m_contractABI = std::move(_contractABI); }
+	void setSideEffects(std::vector<std::string> _sideEffects) { m_call.actualSideEffects = _sideEffects; }
 
 private:
 	/// Tries to format the given `bytes`, applying the detected ABI types that have be set for each parameter.