Refactoring; fuse SyntaxTestParser and SyntaxTester to SyntaxTest.

docs/contributing.rst

@@ -69,15 +69,22 @@ Solidity includes different types of tests. They are included in the application
 called ``soltest``. Some of them require the ``cpp-ethereum`` client in testing mode,
 some others require ``libz3`` to be installed.
 
-To disable the z3 tests, use ``./build/test/soltest -- --no-smt`` and
+``soltest`` reads test contracts that are annotated with expected results
-to run a subset of the tests that do not require ``cpp-ethereum``, use ``./build/test/soltest -- --no-ipc``.
+stored in ``./test/libsolidity/syntaxTests``. In order for soltest to find these
+tests the root test directory has to be specified using the ``--testpath`` command
+line option, e.g. ``./build/test/soltest -- --testpath ./test``.
+
+To disable the z3 tests, use ``./build/test/soltest -- --no-smt --testpath ./test`` and
+to run a subset of the tests that do not require ``cpp-ethereum``, use
+``./build/test/soltest -- --no-ipc --testpath ./test``.
 
 For all other tests, you need to install `cpp-ethereum <https://github.com/ethereum/cpp-ethereum/releases/download/solidityTester/eth>`_ and run it in testing mode: ``eth --test -d /tmp/testeth``.
 
-Then you run the actual tests: ``./build/test/soltest -- --ipcpath /tmp/testeth/geth.ipc``.
+Then you run the actual tests: ``./build/test/soltest -- --ipcpath /tmp/testeth/geth.ipc --testpath ./test``.
 
 To run a subset of tests, filters can be used:
-``soltest -t TestSuite/TestName -- --ipcpath /tmp/testeth/geth.ipc``, where ``TestName`` can be a wildcard ``*``.
+``soltest -t TestSuite/TestName -- --ipcpath /tmp/testeth/geth.ipc --testpath ./test``,
+where ``TestName`` can be a wildcard ``*``.
 
 Alternatively, there is a testing script at ``scripts/test.sh`` which executes all tests and runs
 ``cpp-ethereum`` automatically if it is in the path (but does not download it).

test/TestHelper.h

@@ -35,7 +35,7 @@ namespace test
 struct Options: boost::noncopyable
 {
 	std::string ipcPath;
-	std::string testPath;
+	boost::filesystem::path testPath;
 	bool showMessages = false;
 	bool optimize = false;
 	bool disableIPC = false;

test/boostTest.cpp

@@ -36,7 +36,7 @@
 #pragma GCC diagnostic pop
 
 #include <test/TestHelper.h>
-#include <test/libsolidity/SyntaxTester.h>
+#include <test/libsolidity/SyntaxTest.h>
 
 using namespace boost::unit_test;
 
@@ -55,7 +55,11 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 {
 	master_test_suite_t& master = framework::master_test_suite();
 	master.p_name.value = "SolidityTests";
-	dev::solidity::test::SyntaxTester::registerTests();
+	solAssert(dev::solidity::test::SyntaxTest::registerTests(
+		master,
+		dev::test::Options::get().testPath / "libsolidity",
+		"syntaxTests"
+	) > 0, "no syntax tests found");
 	if (dev::test::Options::get().disableIPC)
 	{
 		for (auto suite: {

test/libsolidity/SyntaxTest.cpp

@@ -0,0 +1,205 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SyntaxTest.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/throw_exception.hpp>
+#include <cctype>
+#include <fstream>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+namespace fs = boost::filesystem;
+using namespace boost::unit_test;
+
+template<typename IteratorType>
+void skipWhitespace(IteratorType& it, IteratorType end)
+{
+	while (it != end && isspace(*it))
+		++it;
+}
+
+template<typename IteratorType>
+void skipSlashes(IteratorType& it, IteratorType end)
+{
+	while (it != end && *it == '/')
+		++it;
+}
+
+SyntaxTest::SyntaxTest(string const& _filename)
+{
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\""));
+	file.exceptions(ios::badbit);
+
+	m_source = parseSource(file);
+	m_expectations = parseExpectations(file);
+}
+
+bool SyntaxTest::run(ostream& _stream, string const& _indent)
+{
+	m_errorList = parseAnalyseAndReturnError(m_source, true, true, true).second;
+	if (!matchesExpectations(m_errorList))
+	{
+		std::string nextIndentLevel = _indent.empty() ? "\t" : _indent + _indent;
+		_stream << _indent << "Expected result:" << endl;
+		printExpected(_stream, nextIndentLevel);
+		_stream << _indent << "Obtained result:\n";
+		printErrorList(_stream, m_errorList, nextIndentLevel);
+		return false;
+	}
+	return true;
+}
+
+void SyntaxTest::printExpected(ostream& _stream, string const& _indent) const
+{
+	if (m_expectations.empty())
+		_stream << _indent << "Success" << endl;
+	else
+		for (auto const& expectation: m_expectations)
+			_stream << _indent << expectation.type << ": " << expectation.message << endl;
+}
+
+void SyntaxTest::printErrorList(
+	ostream& _stream,
+	ErrorList const& _errorList,
+	string const& _indent
+) const
+{
+	if (_errorList.empty())
+		_stream << _indent << "Success" << endl;
+	else
+		for (auto const& error: _errorList)
+			_stream << _indent << error->typeName() << ": " << errorMessage(*error) << endl;
+}
+
+bool SyntaxTest::matchesExpectations(ErrorList const& _errorList) const
+{
+	if (_errorList.size() != m_expectations.size())
+		return false;
+	else
+		for (size_t i = 0; i < _errorList.size(); i++)
+			if (
+				!(_errorList[i]->typeName() == m_expectations[i].type) ||
+				!(errorMessage(*_errorList[i]) == m_expectations[i].message)
+			)
+				return false;
+	return true;
+}
+
+string SyntaxTest::errorMessage(Error const& _e)
+{
+	if (_e.comment())
+		return boost::replace_all_copy(*_e.comment(), "\n", "\\n");
+	else
+		return "NONE";
+}
+
+string SyntaxTest::parseSource(istream& _stream)
+{
+	string source;
+	string line;
+	string const delimiter("// ----");
+	while (getline(_stream, line))
+		if (boost::algorithm::starts_with(line, delimiter))
+			break;
+		else
+			source += line + "\n";
+	return source;
+}
+
+vector<SyntaxTestExpectation> SyntaxTest::parseExpectations(istream& _stream)
+{
+	vector<SyntaxTestExpectation> expectations;
+	string line;
+	while (getline(_stream, line))
+	{
+		auto it = line.begin();
+
+		skipSlashes(it, line.end());
+		skipWhitespace(it, line.end());
+
+		if (it == line.end()) continue;
+
+		auto typeBegin = it;
+		while (it != line.end() && *it != ':')
+			++it;
+		string errorType(typeBegin, it);
+
+		// skip colon
+		if (it != line.end()) it++;
+
+		skipWhitespace(it, line.end());
+
+		string errorMessage(it, line.end());
+		expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)});
+	}
+	return expectations;
+}
+
+#if BOOST_VERSION < 105900
+test_case *make_test_case(
+	function<void()> const& _fn,
+	string const& _name,
+	string const& /* _filename */,
+	size_t /* _line */
+)
+{
+	return make_test_case(_fn, _name);
+}
+#endif
+
+int SyntaxTest::registerTests(
+	boost::unit_test::test_suite& _suite,
+	boost::filesystem::path const& _basepath,
+	boost::filesystem::path const& _path
+)
+{
+	int numTestsAdded = 0;
+	fs::path fullpath = _basepath / _path;
+	if (fs::is_directory(fullpath))
+	{
+		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
+		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
+			fs::directory_iterator(fullpath),
+			fs::directory_iterator()
+		))
+			numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
+		_suite.add(sub_suite);
+	}
+	else
+	{
+		_suite.add(make_test_case(
+			[fullpath]
+			{
+				std::stringstream errorStream;
+				if (!SyntaxTest(fullpath.string()).run(errorStream, ""))
+					BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
+			},
+			_path.stem().string(),
+			_path.string(),
+			0
+		));
+		numTestsAdded = 1;
+	}
+	return numTestsAdded;
+}

test/libsolidity/SyntaxTest.h

@@ -18,10 +18,16 @@
 #pragma once
 
 #include <test/libsolidity/AnalysisFramework.h>
-#include <test/libsolidity/SyntaxTestParser.h>
+#include <libsolidity/interface/Exceptions.h>
-#include <boost/filesystem.hpp>
+
+#include <boost/noncopyable.hpp>
 #include <boost/test/unit_test.hpp>
 
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
 namespace dev
 {
 namespace solidity
@@ -29,18 +35,41 @@ namespace solidity
 namespace test
 {
 
-class SyntaxTester: public AnalysisFramework
+struct SyntaxTestExpectation
+{
+	std::string type;
+	std::string message;
+};
+
+
+class SyntaxTest: AnalysisFramework
 {
 public:
-	static void registerTests();
+	SyntaxTest(std::string const& _filename);
-private:
+
+	bool run(std::ostream& _stream, std::string const& _indent);
+
+	void printExpected(std::ostream& _stream, std::string const& _indent) const;
+	void printErrorList(
+		std::ostream& _stream,
+		ErrorList const& _errors,
+		std::string const& _indent
+	) const;
+
 	static int registerTests(
 		boost::unit_test::test_suite& _suite,
 		boost::filesystem::path const& _basepath,
 		boost::filesystem::path const& _path
 	);
+private:
+	bool matchesExpectations(ErrorList const& _errors) const;
 	static std::string errorMessage(Error const& _e);
-	void runTest(SyntaxTest const& _test);
+	static std::string parseSource(std::istream& _stream);
+	static std::vector<SyntaxTestExpectation> parseExpectations(std::istream& _stream);
+
+	std::string m_source;
+	std::vector<SyntaxTestExpectation> m_expectations;
+	ErrorList m_errorList;
 };
 
 }

test/libsolidity/SyntaxTestParser.cpp

@@ -1,97 +0,0 @@
-/*
-	This file is part of solidity.
-
-	solidity is free software: you can redistribute it and/or modify
-	it under the terms of the GNU General Public License as published by
-	the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-
-	solidity is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <test/libsolidity/SyntaxTestParser.h>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/throw_exception.hpp>
-#include <cctype>
-#include <fstream>
-#include <stdexcept>
-
-using namespace dev;
-using namespace solidity;
-using namespace dev::solidity::test;
-using namespace std;
-
-template<typename IteratorType>
-void skipWhitespace(IteratorType& it, IteratorType end)
-{
-	while (it != end && isspace(*it))
-		++it;
-}
-
-template<typename IteratorType>
-void skipSlashes(IteratorType& it, IteratorType end)
-{
-	while (it != end && *it == '/')
-		++it;
-}
-
-std::string SyntaxTestParser::parseSource(std::istream& _stream)
-{
-	std::string source;
-	string line;
-	string const delimiter("// ----");
-	while (getline(_stream, line))
-		if (boost::algorithm::starts_with(line, delimiter))
-			break;
-		else
-			source += line + "\n";
-	return source;
-}
-
-std::vector<SyntaxTestExpectation> SyntaxTestParser::parseExpectations(std::istream& _stream)
-{
-	std::vector<SyntaxTestExpectation> expectations;
-	std::string line;
-	while (getline(_stream, line))
-	{
-		auto it = line.begin();
-
-		skipSlashes(it, line.end());
-		skipWhitespace(it, line.end());
-
-		if (it == line.end()) continue;
-
-		auto typeBegin = it;
-		while (it != line.end() && *it != ':')
-			++it;
-		string errorType(typeBegin, it);
-
-		// skip colon
-		if (it != line.end()) it++;
-
-		skipWhitespace(it, line.end());
-
-		string errorMessage(it, line.end());
-		expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)});
-	}
-	return expectations;
-}
-
-SyntaxTest SyntaxTestParser::parse(string const& _filename)
-{
-	ifstream file(_filename);
-	if (!file)
-		BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\""));
-	file.exceptions(ios::badbit);
-
-	SyntaxTest result;
-	result.source = parseSource(file);
-	result.expectations = parseExpectations(file);
-	return result;
-}

test/libsolidity/SyntaxTestParser.h

@@ -1,57 +0,0 @@
-/*
-	This file is part of solidity.
-
-	solidity is free software: you can redistribute it and/or modify
-	it under the terms of the GNU General Public License as published by
-	the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-
-	solidity is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#pragma once
-
-#include <iosfwd>
-#include <string>
-#include <vector>
-#include <utility>
-
-namespace dev
-{
-namespace solidity
-{
-namespace test
-{
-
-struct SyntaxTestExpectation
-{
-	std::string type;
-	std::string message;
-};
-
-struct SyntaxTest
-{
-	std::string source;
-	std::vector<SyntaxTestExpectation> expectations;
-};
-
-class SyntaxTestParser
-{
-public:
-	SyntaxTestParser() = default;
-
-	SyntaxTest parse(std::string const& _filename);
-private:
-	std::string parseSource(std::istream& _stream);
-	std::vector<SyntaxTestExpectation> parseExpectations(std::istream& _stream);
-};
-
-}
-}
-}

test/libsolidity/SyntaxTester.cpp

@@ -1,147 +0,0 @@
-/*
-	This file is part of solidity.
-
-	solidity is free software: you can redistribute it and/or modify
-	it under the terms of the GNU General Public License as published by
-	the Free Software Foundation, either version 3 of the License, or
-	(at your option) any later version.
-
-	solidity is distributed in the hope that it will be useful,
-	but WITHOUT ANY WARRANTY; without even the implied warranty of
-	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-	GNU General Public License for more details.
-
-	You should have received a copy of the GNU General Public License
-	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include <test/libsolidity/SyntaxTester.h>
-#include <test/libsolidity/AnalysisFramework.h>
-#include <test/TestHelper.h>
-#include <boost/algorithm/string/replace.hpp>
-
-using namespace dev;
-using namespace solidity;
-using namespace dev::solidity::test;
-using namespace std;
-using namespace boost::unit_test;
-namespace fs = boost::filesystem;
-
-#if BOOST_VERSION < 105900
-test_case *make_test_case(
-	function<void()> const& _fn,
-	string const& _name,
-	string const&, // _filename
-	size_t // _line
-)
-{
-	return make_test_case(_fn, _name);
-}
-#endif
-
-
-void SyntaxTester::runTest(SyntaxTest const& _test)
-{
-	vector<string> unexpectedErrors;
-	auto expectations = _test.expectations;
-	auto errorList = parseAnalyseAndReturnError(_test.source, true, true, true).second;
-
-	bool errorsMatch = true;
-
-	if (errorList.size() != expectations.size())
-		errorsMatch = false;
-	else
-	{
-		for (size_t i = 0; i < errorList.size(); i++)
-		{
-			if (
-				!(errorList[i]->typeName() == expectations[i].type) ||
-				!(errorMessage(*errorList[i]) == expectations[i].message)
-			)
-			{
-				errorsMatch = false;
-				break;
-			}
-		}
-	}
-
-	if (!errorsMatch)
-	{
-		string msg = "Test expectation mismatch.\nExpected result:\n";
-		if (expectations.empty())
-			msg += "\tSuccess\n";
-		else
-			for (auto const& expectation: expectations)
-				msg += "\t" + expectation.type + ": " + expectation.message + "\n";
-		msg += "Obtained result:\n";
-		if (errorList.empty())
-			msg += "\tSuccess\n";
-		else
-			for (auto const& error: errorList)
-				msg += "\t" + error->typeName() + ": " + errorMessage(*error) + "\n";
-		BOOST_ERROR(msg);
-	}
-}
-
-std::string SyntaxTester::errorMessage(Error const& _e)
-{
-	if (_e.comment())
-		return boost::replace_all_copy(*_e.comment(), "\n", "\\n");
-	else
-		return "NONE";
-}
-
-int SyntaxTester::registerTests(
-	test_suite& _suite,
-	fs::path const& _basepath,
-	fs::path const& _path
-)
-{
-
-	int numTestsAdded = 0;
-	fs::path fullpath = _basepath / _path;
-	if (fs::is_directory(fullpath))
-	{
-		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
-		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
-			fs::directory_iterator(fullpath),
-			fs::directory_iterator()
-		))
-			numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
-		_suite.add(sub_suite);
-	}
-	else
-	{
-		_suite.add(make_test_case(
-			[fullpath] { SyntaxTester().runTest(SyntaxTestParser().parse(fullpath.string())); },
-			_path.stem().string(),
-			_path.string(),
-			0
-		));
-		numTestsAdded = 1;
-	}
-	return numTestsAdded;
-}
-
-void SyntaxTester::registerTests()
-{
-	if(dev::test::Options::get().testPath.empty())
-		throw runtime_error(
-			"No path to the test files was specified. "
-			"Use the --testpath command line option or "
-			"the ETH_TEST_PATH environment variable."
-		);
-	auto testPath = fs::path(dev::test::Options::get().testPath);
-
-	if (fs::exists(testPath) && fs::is_directory(testPath))
-	{
-		int numTestsAdded = registerTests(
-			framework::master_test_suite(),
-			testPath / "libsolidity",
-			"syntaxTests"
-		);
-		solAssert(numTestsAdded > 0, "no syntax tests found in libsolidity/syntaxTests");
-	}
-	else
-		solAssert(false, "libsolidity/syntaxTests directory not found");
-}