Implements infrastructure for semantic tests.

test/ExecutionFramework.cpp

@@ -49,8 +49,13 @@ string getIPCSocketPath()
 
 }
 
-ExecutionFramework::ExecutionFramework() :
+ExecutionFramework::ExecutionFramework():
-	m_rpc(RPCSession::instance(getIPCSocketPath())),
+	ExecutionFramework(getIPCSocketPath())
+{
+}
+
+ExecutionFramework::ExecutionFramework(string const& _ipcPath):
+	m_rpc(RPCSession::instance(_ipcPath)),
 	m_evmVersion(dev::test::Options::get().evmVersion()),
 	m_optimize(dev::test::Options::get().optimize),
 	m_showMessages(dev::test::Options::get().showMessages),

test/ExecutionFramework.h

@@ -53,6 +53,7 @@ class ExecutionFramework
 
 public:
 	ExecutionFramework();
+	explicit ExecutionFramework(std::string const& _ipcPath);
 	virtual ~ExecutionFramework() = default;
 
 	virtual bytes const& compileAndRunWithoutCheck(

test/InteractiveTests.h

@@ -20,6 +20,7 @@
 #include <test/TestCase.h>
 #include <test/libsolidity/ASTJSONTest.h>
 #include <test/libsolidity/SyntaxTest.h>
+#include <test/libsolidity/SemanticTest.h>
 #include <test/libsolidity/SMTCheckerJSONTest.h>
 #include <test/libyul/YulOptimizerTest.h>
 #include <test/libyul/ObjectCompilerTest.h>
@@ -52,6 +53,7 @@ Testsuite const g_interactiveTestsuites[] = {
 	{"Yul Optimizer",       "libyul",      "yulOptimizerTests",   false, false, &yul::test::YulOptimizerTest::create},
 	{"Yul Object Compiler", "libyul",      "objectCompiler",      false, false, &yul::test::ObjectCompilerTest::create},
 	{"Syntax",              "libsolidity", "syntaxTests",         false, false, &SyntaxTest::create},
+	{"Semantic",            "libsolidity", "semanticTests",       false, true,  &SemanticTest::create},
 	{"JSON AST",            "libsolidity", "ASTJSON",             false, false, &ASTJSONTest::create},
 	{"SMT Checker",         "libsolidity", "smtCheckerTests",     true,  false, &SyntaxTest::create},
 	{"SMT Checker JSON",    "libsolidity", "smtCheckerTestsJSON", true,  false, &SMTCheckerTest::create}

test/TestCase.h

@@ -34,7 +34,13 @@ namespace test
 class TestCase
 {
 public:
-	using TestCaseCreator = std::unique_ptr<TestCase>(*)(std::string const&);
+	struct Config
+	{
+		std::string filename;
+		std::string ipcPath;
+	};
+
+	using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
 
 	virtual ~TestCase() = default;
 

test/libsolidity/ASTJSONTest.h

@@ -35,8 +35,8 @@ namespace test
 class ASTJSONTest: public TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
-	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_filename)); }
+	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
 	ASTJSONTest(std::string const& _filename);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;

test/libsolidity/SMTCheckerJSONTest.h

@@ -33,9 +33,9 @@ namespace test
 class SMTCheckerTest: public SyntaxTest
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new SMTCheckerTest(_filename));
+		return std::unique_ptr<TestCase>(new SMTCheckerTest(_config.filename));
 	}
 	SMTCheckerTest(std::string const& _filename);
 

test/libsolidity/SemanticTest.cpp

@@ -0,0 +1,222 @@
+/*
+	This file is part of solidity.
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SemanticTest.h>
+#include <test/Options.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/trim.hpp>
+#include <boost/throw_exception.hpp>
+
+#include <algorithm>
+#include <cctype>
+#include <fstream>
+#include <memory>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace dev::solidity::test::formatting;
+using namespace std;
+namespace fs = boost::filesystem;
+using namespace boost;
+using namespace boost::algorithm;
+using namespace boost::unit_test;
+
+namespace
+{
+	using ParamList = dev::solidity::test::ParameterList;
+	using FunctionCallTest = dev::solidity::test::SemanticTest::FunctionCallTest;
+	using FunctionCall = dev::solidity::test::FunctionCall;
+
+	string formatBytes(bytes const& _bytes, ParamList const& _params, bool const _formatInvalid = false)
+	{
+		stringstream resultStream;
+		if (_bytes.empty())
+			resultStream.str();
+		auto it = _bytes.begin();
+		for (auto const& param: _params)
+		{
+			bytes byteRange{it, it + param.abiType.size};
+			// FIXME Check range
+			// TODO Check range
+			switch (param.abiType.type)
+			{
+			case ABIType::SignedDec:
+				if (*byteRange.begin() & 0x80)
+					resultStream << u2s(fromBigEndian<u256>(byteRange));
+				else
+					resultStream << fromBigEndian<u256>(byteRange);
+				break;
+			case ABIType::UnsignedDec:
+				// Check if the detected type was wrong and if this could
+				// be signed. If an unsigned was detected in the expectations,
+				// but the actual result returned a signed, it would be formatted
+				// incorrectly.
+				if (*byteRange.begin() & 0x80)
+					resultStream << u2s(fromBigEndian<u256>(byteRange));
+				else
+					resultStream << fromBigEndian<u256>(byteRange);
+				break;
+			case ABIType::Failure:
+				// If expectations are empty, the encoding type is invalid.
+				// In order to still print the actual result even if
+				// empty expectations were detected, it must be forced.
+				if (_formatInvalid)
+					resultStream << fromBigEndian<u256>(byteRange);
+				break;
+			case ABIType::None:
+				// If expectations are empty, the encoding type is NONE.
+				if (_formatInvalid)
+					resultStream << fromBigEndian<u256>(byteRange);
+				break;
+			}
+			it += param.abiType.size;
+			if (it != _bytes.end() && !(param.abiType.type == ABIType::None))
+				resultStream << ", ";
+		}
+		return resultStream.str();
+	}
+
+	string formatFunctionCallTest(
+		FunctionCallTest const& _test,
+		string const& _linePrefix = "",
+		bool const _renderResult = false,
+		bool const _higlight = false
+	)
+	{
+		stringstream _stream;
+		FunctionCall call = _test.call;
+		bool hightlight = !_test.matchesExpectation() && _higlight;
+
+		auto formatOutput = [&](bool const _singleLine)
+		{
+			_stream << _linePrefix << "// " << call.signature;
+			if (call.value > u256(0))
+				_stream << TestFileParser::formatToken(SoltToken::Comma)
+						<< call.value << " "
+						<< TestFileParser::formatToken(SoltToken::Ether);
+			if (!call.arguments.rawBytes().empty())
+				_stream << ": "
+						<< formatBytes(call.arguments.rawBytes(), call.arguments.parameters);
+			if (!_singleLine)
+				_stream << endl << _linePrefix << "// ";
+			if (_singleLine)
+				_stream << " ";
+			_stream << "-> ";
+			if (!_singleLine)
+				_stream << endl << _linePrefix << "// ";
+			if (hightlight)
+				_stream << formatting::RED_BACKGROUND;
+			bytes output;
+			if (_renderResult)
+				output = call.expectations.rawBytes();
+			else
+				output = _test.rawBytes;
+			if (!output.empty())
+				_stream << formatBytes(output, call.expectations.result);
+			if (hightlight)
+				_stream << formatting::RESET;
+		};
+
+		if (call.displayMode == FunctionCall::DisplayMode::SingleLine)
+			formatOutput(true);
+		else
+			formatOutput(false);
+		_stream << endl;
+
+		return _stream.str();
+	}
+}
+
+SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
+	SolidityExecutionFramework(_ipcPath)
+{
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\"."));
+	file.exceptions(ios::badbit);
+
+	m_source = parseSource(file);
+	parseExpectations(file);
+}
+
+bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+{
+	if (!deploy("", 0, bytes()))
+		BOOST_THROW_EXCEPTION(runtime_error("Failed to deploy contract."));
+
+	bool success = true;
+	for (auto& test: m_tests)
+		test.reset();
+
+	for (auto& test: m_tests)
+	{
+		bytes output = callContractFunctionWithValueNoEncoding(
+			test.call.signature,
+			test.call.value,
+			test.call.arguments.rawBytes()
+		);
+
+		if ((m_transactionSuccessful == test.call.expectations.failure) || (output != test.call.expectations.rawBytes()))
+			success = false;
+
+		test.failure = !m_transactionSuccessful;
+		test.rawBytes = std::move(output);
+	}
+
+	if (!success)
+	{
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
+		for (auto const& test: m_tests)
+			_stream << formatFunctionCallTest(test, _linePrefix, false, true);
+
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
+		for (auto const& test: m_tests)
+			_stream << formatFunctionCallTest(test, _linePrefix, true, true);
+
+		FormattedScope(_stream, _formatted, {BOLD, RED}) << _linePrefix
+			<< "Attention: Updates on the test will apply the detected format displayed." << endl;
+		return false;
+	}
+	return true;
+}
+
+void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
+{
+	stringstream stream(m_source);
+	string line;
+	while (getline(stream, line))
+		_stream << _linePrefix << line << endl;
+}
+
+void SemanticTest::printUpdatedExpectations(ostream& _stream, string const& _linePrefix) const
+{
+	for (auto const& test: m_tests)
+		_stream << formatFunctionCallTest(test, _linePrefix, false, false);
+}
+
+void SemanticTest::parseExpectations(istream& _stream)
+{
+	TestFileParser parser{_stream};
+	for (auto const& call: parser.parseFunctionCalls())
+		m_tests.emplace_back(FunctionCallTest{call, bytes{}, string{}});
+}
+
+bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments)
+{
+	auto output = compileAndRunWithoutCheck(m_source, _value, _contractName, _arguments);
+	return !output.empty() && m_transactionSuccessful;
+}

test/libsolidity/SemanticTest.h

@@ -0,0 +1,99 @@
+/*
+	This file is part of solidity.
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/util/TestFileParser.h>
+#include <test/libsolidity/FormattedScope.h>
+#include <test/libsolidity/SolidityExecutionFramework.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/TestCase.h>
+#include <liblangutil/Exceptions.h>
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+/**
+ * Class that represents a semantic test (or end-to-end test) and allows running it as part of the
+ * boost unit test environment or isoltest. It reads the Solidity source and an additional comment
+ * section from the given file. This comment section should define a set of functions to be called
+ * and an expected result they return after being executed.
+ */
+class SemanticTest: public SolidityExecutionFramework, public TestCase
+{
+public:
+	/**
+	 * Represents a function call and the result it returned. It stores the call
+	 * representation itself, the actual byte result (if any) and a string representation
+	 * used for the interactive update routine provided by isoltest. It also provides
+	 * functionality to compare the actual result with the expectations attached to the
+	 * call object, as well as a way to reset the result if executed multiple times.
+	 */
+	struct FunctionCallTest
+	{
+		FunctionCall call;
+		bytes rawBytes;
+		std::string output;
+		bool failure = true;
+		/// Compares raw expectations (which are converted to a byte representation before),
+		/// and also the expected transaction status of the function call to the actual test results.
+		bool matchesExpectation() const
+		{
+			return failure == call.expectations.failure && rawBytes == call.expectations.rawBytes();
+		}
+		/// Resets current results in case the function was called and the result
+		/// stored already (e.g. if test case was updated via isoltest).
+		void reset()
+		{
+			failure = true;
+			rawBytes = bytes{};
+			output = std::string{};
+		}
+	};
+
+	static std::unique_ptr<TestCase> create(Config const& _options)
+	{ return std::make_unique<SemanticTest>(_options.filename, _options.ipcPath); }
+
+	explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath);
+
+	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
+	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;
+
+	/// Instantiates a test file parser that parses the additional comment section at the end of
+	/// the input stream \param _stream. Each function call is represented using a `FunctionCallTest`
+	/// and added to the list of call to be executed when `run()` is called.
+	/// Throws if parsing expectations failed.
+	void parseExpectations(std::istream& _stream);
+
+	/// Compiles and deploys currently held source.
+	/// Returns true if deployment was successful, false otherwise.
+	bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments);
+
+	std::string m_source;
+	std::vector<FunctionCallTest> m_tests;
+};
+
+}
+}
+}

test/libsolidity/SolidityExecutionFramework.cpp

@@ -28,7 +28,12 @@ using namespace dev::test;
 using namespace dev::solidity;
 using namespace dev::solidity::test;
 
-SolidityExecutionFramework::SolidityExecutionFramework() :
+SolidityExecutionFramework::SolidityExecutionFramework():
 	ExecutionFramework()
 {
 }
+
+SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath):
+	ExecutionFramework(_ipcPath)
+{
+}

test/libsolidity/SolidityExecutionFramework.h

@@ -43,6 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework
 
 public:
 	SolidityExecutionFramework();
+	SolidityExecutionFramework(std::string const& _ipcPath);
 
 	virtual bytes const& compileAndRunWithoutCheck(
 		std::string const& _sourceCode,

test/libsolidity/SyntaxTest.h

@@ -53,8 +53,8 @@ struct SyntaxTestError
 class SyntaxTest: AnalysisFramework, public TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
-	{ return std::unique_ptr<TestCase>(new SyntaxTest(_filename)); }
+	{ return std::unique_ptr<TestCase>(new SyntaxTest(_config.filename)); }
 	SyntaxTest(std::string const& _filename);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;

test/libyul/ObjectCompilerTest.h

@@ -40,9 +40,9 @@ namespace test
 class ObjectCompilerTest: public dev::solidity::test::TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new ObjectCompilerTest(_filename));
+		return std::unique_ptr<TestCase>(new ObjectCompilerTest(_config.filename));
 	}
 
 	explicit ObjectCompilerTest(std::string const& _filename);

test/libyul/YulOptimizerTest.h

@@ -41,9 +41,9 @@ namespace test
 class YulOptimizerTest: public dev::solidity::test::TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new YulOptimizerTest(_filename));
+		return std::unique_ptr<TestCase>(new YulOptimizerTest(_config.filename));
 	}
 
 	explicit YulOptimizerTest(std::string const& _filename);

test/tools/CMakeLists.txt

@@ -13,7 +13,9 @@ add_executable(isoltest
 	../Options.cpp
 	../Common.cpp
 	../TestCase.cpp
+	../libsolidity/util/TestFileParser.cpp
 	../libsolidity/SyntaxTest.cpp
+        ../libsolidity/SemanticTest.cpp
 	../libsolidity/AnalysisFramework.cpp
 	../libsolidity/SolidityExecutionFramework.cpp
 	../ExecutionFramework.cpp