Allow test cases to indicate fatal errors.

test/TestCase.h

@@ -56,6 +56,8 @@ public:
 		langutil::EVMVersion evmVersion;
 	};
 
+	enum class TestResult { Success, Failure, FatalError };
+
 	using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
 
 	virtual ~TestCase() = default;
@@ -64,7 +66,7 @@ public:
 	/// Outputs error messages to @arg _stream. Each line of output is prefixed with @arg _linePrefix.
 	/// Optionally, color-coding can be enabled (if @arg _formatted is set to true).
 	/// @returns true, if the test case succeeds, false otherwise
-	virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
+	virtual TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
 
 	/// Outputs the test contract to @arg _stream.
 	/// Each line of output is prefixed with @arg _linePrefix.

test/boostTest.cpp

@@ -106,8 +106,17 @@ int registerTests(
 						stringstream errorStream;
 						auto testCase = _testCaseCreator(config);
 						if (testCase->validateSettings(dev::test::Options::get().evmVersion()))
-							if (!testCase->run(errorStream))
+							switch (testCase->run(errorStream))
-								BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
+							{
+								case TestCase::TestResult::Success:
+									break;
+								case TestCase::TestResult::Failure:
+									BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
+									break;
+								case TestCase::TestResult::FatalError:
+									BOOST_ERROR("Fatal error during test.\n" + errorStream.str());
+									break;
+							}
 					}
 					catch (boost::exception const& _e)
 					{

test/libsolidity/ASTJSONTest.cpp

@@ -18,6 +18,7 @@
 #include <test/libsolidity/ASTJSONTest.h>
 #include <test/Options.h>
 #include <libdevcore/AnsiColorized.h>
+#include <liblangutil/SourceReferenceFormatterHuman.h>
 #include <libsolidity/ast/ASTJsonConverter.h>
 #include <libsolidity/interface/CompilerStack.h>
 #include <boost/algorithm/string.hpp>
@@ -27,6 +28,7 @@
 #include <memory>
 #include <stdexcept>
 
+using namespace langutil;
 using namespace dev::solidity;
 using namespace dev::solidity::test;
 using namespace dev::formatting;
@@ -88,7 +90,7 @@ ASTJSONTest::ASTJSONTest(string const& _filename)
 	}
 }
 
-bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	CompilerStack c;
 
@@ -101,7 +103,15 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 	}
 	c.setSources(sources);
 	c.setEVMVersion(dev::test::Options::get().evmVersion());
-	c.parseAndAnalyze();
+	if (c.parse())
+		c.analyze();
+	else
+	{
+		SourceReferenceFormatterHuman formatter(_stream, _formatted);
+		for (auto const& error: c.errors())
+			formatter.printErrorInformation(*error);
+		return TestResult::FatalError;
+	}
 
 	for (size_t i = 0; i < m_sources.size(); i++)
 	{
@@ -169,7 +179,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
 		resultsMatch = false;
 	}
 
-	return resultsMatch;
+	return resultsMatch ? TestResult::Success : TestResult::Failure;
 }
 
 void ASTJSONTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libsolidity/ASTJSONTest.h

@@ -39,7 +39,7 @@ public:
 	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
 	ASTJSONTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libsolidity/GasTest.cpp

@@ -158,7 +158,7 @@ void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const&
 }
 
 
-bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	string const versionPragma = "pragma solidity >=0.0;\n";
 	compiler().reset();
@@ -174,10 +174,10 @@ bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 
 	if (!compiler().parseAndAnalyze() || !compiler().compile())
 	{
-		SourceReferenceFormatterHuman formatter(cerr, _formatted);
+		SourceReferenceFormatterHuman formatter(_stream, _formatted);
 		for (auto const& error: compiler().errors())
 			formatter.printErrorInformation(*error);
-		BOOST_THROW_EXCEPTION(runtime_error("Test contract does not compile."));
+		return TestResult::FatalError;
 	}
 
 	Json::Value estimates = compiler().gasEstimates(compiler().lastContractName());
@@ -226,7 +226,7 @@ bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 		printUpdatedExpectations(_stream, _linePrefix + "  ");
 	}
 
-	return success;
+	return success ? TestResult::Success : TestResult::Failure;
 }
 
 void GasTest::printSource(ostream& _stream, string const& _linePrefix, bool) const

test/libsolidity/GasTest.h

@@ -40,7 +40,7 @@ public:
 	{ return std::make_unique<GasTest>(_config.filename); }
 	GasTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
 	void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libsolidity/SMTCheckerJSONTest.cpp

@@ -49,7 +49,7 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev
 		BOOST_THROW_EXCEPTION(runtime_error("Invalid JSON file."));
 }
 
-bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	StandardCompiler compiler;
 
@@ -121,7 +121,7 @@ bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _form
 		}
 	}
 
-	return printExpectationAndError(_stream, _linePrefix, _formatted);
+	return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
 }
 
 vector<string> SMTCheckerTest::hashesFromJson(Json::Value const& _jsonObj, string const& _auxInput, string const& _smtlib)

test/libsolidity/SMTCheckerJSONTest.h

@@ -39,7 +39,7 @@ public:
 	}
 	SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
 private:
 	std::vector<std::string> hashesFromJson(Json::Value const& _jsonObj, std::string const& _auxInput, std::string const& _smtlib);

test/libsolidity/SemanticTest.cpp

@@ -53,7 +53,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, lang
 	parseExpectations(file);
 }
 
-bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
 
@@ -96,9 +96,9 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _format
 		}
 		AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix << endl << _linePrefix
 			<< "Attention: Updates on the test will apply the detected format displayed." << endl;
-		return false;
+		return TestResult::Failure;
 	}
-	return true;
+	return TestResult::Success;
 }
 
 void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const

test/libsolidity/SemanticTest.h

@@ -48,7 +48,7 @@ public:
 
 	explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;
 

test/libsolidity/SyntaxTest.cpp

@@ -63,7 +63,7 @@ SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion
 	m_expectations = parseExpectations(file);
 }
 
-bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
+TestCase::TestResult SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	string const versionPragma = "pragma solidity >=0.0;\n";
 	compiler().reset();
@@ -92,7 +92,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatte
 		});
 	}
 
-	return printExpectationAndError(_stream, _linePrefix, _formatted);
+	return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
 }
 
 bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool _formatted)

test/libsolidity/SyntaxTest.h

@@ -57,7 +57,7 @@ public:
 	{ return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion); }
 	SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
 	void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override

test/libyul/ObjectCompilerTest.cpp

@@ -62,7 +62,7 @@ ObjectCompilerTest::ObjectCompilerTest(string const& _filename)
 			m_expectation += line + "\n";
 }
 
-bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	AssemblyStack stack(
 		EVMVersion(),
@@ -73,7 +73,7 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
 	{
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Error parsing source." << endl;
 		printErrors(_stream, stack.errors());
-		return false;
+		return TestResult::FatalError;
 	}
 	stack.optimize();
 
@@ -98,9 +98,9 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
 		printIndented(_stream, m_expectation, nextIndentLevel);
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
 		printIndented(_stream, m_obtainedResult, nextIndentLevel);
-		return false;
+		return TestResult::Failure;
 	}
-	return true;
+	return TestResult::Success;
 }
 
 void ObjectCompilerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libyul/ObjectCompilerTest.h

@@ -47,7 +47,7 @@ public:
 
 	explicit ObjectCompilerTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libyul/YulInterpreterTest.cpp

@@ -67,10 +67,10 @@ YulInterpreterTest::YulInterpreterTest(string const& _filename)
 			m_expectation += line + "\n";
 }
 
-bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	if (!parse(_stream, _linePrefix, _formatted))
-		return false;
+		return TestResult::FatalError;
 
 	m_obtainedResult = interpret();
 
@@ -82,9 +82,9 @@ bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool c
 		printIndented(_stream, m_expectation, nextIndentLevel);
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
 		printIndented(_stream, m_obtainedResult, nextIndentLevel);
-		return false;
+		return TestResult::Failure;
 	}
-	return true;
+	return TestResult::Success;
 }
 
 void YulInterpreterTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libyul/YulInterpreterTest.h

@@ -47,7 +47,7 @@ public:
 
 	explicit YulInterpreterTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

test/libyul/YulOptimizerTest.cpp

@@ -103,10 +103,10 @@ YulOptimizerTest::YulOptimizerTest(string const& _filename)
 			m_expectation += line + "\n";
 }
 
-bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
 {
 	if (!parse(_stream, _linePrefix, _formatted))
-		return false;
+		return TestResult::FatalError;
 
 	if (m_optimizerStep == "disambiguator")
 		disambiguate();
@@ -277,12 +277,11 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
 	else
 	{
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Invalid optimizer step: " << m_optimizerStep << endl;
-		return false;
+		return TestResult::FatalError;
 	}
 
 	m_obtainedResult = AsmPrinter{m_yul}(*m_ast) + "\n";
 
-	bool success = true;
 	if (m_optimizerStep != m_validatedSettings["step"])
 	{
 		string nextIndentLevel = _linePrefix + "  ";
@@ -294,7 +293,7 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
 			m_optimizerStep <<
 			"\"." <<
 			endl;
-		success = false;
+		return TestResult::FatalError;
 	}
 	if (m_expectation != m_obtainedResult)
 	{
@@ -304,9 +303,9 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
 		printIndented(_stream, m_expectation, nextIndentLevel);
 		AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
 		printIndented(_stream, m_obtainedResult, nextIndentLevel);
-		success = false;
+		return TestResult::Failure;
 	}
-	return success;
+	return TestResult::Success;
 }
 
 void YulOptimizerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

test/libyul/YulOptimizerTest.h

@@ -48,7 +48,7 @@ public:
 
 	explicit YulOptimizerTest(std::string const& _filename);
 
-	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
 	void printUpdatedSettings(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) override;

test/tools/isoltest.cpp

@@ -147,7 +147,6 @@ bool TestTool::m_exitRequested = false;
 
 TestTool::Result TestTool::process()
 {
-	bool success;
 	bool formatted{!m_options.noColor};
 	std::stringstream outputMessages;
 
@@ -159,7 +158,21 @@ TestTool::Result TestTool::process()
 
 			m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
 			if (m_test->validateSettings(m_options.evmVersion()))
-				success = m_test->run(outputMessages, "  ", formatted);
+				switch (TestCase::TestResult result = m_test->run(outputMessages, "  ", formatted))
+				{
+					case TestCase::TestResult::Success:
+						AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
+						return Result::Success;
+					default:
+						AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
+
+						AnsiColorized(cout, formatted, {BOLD, CYAN}) << "  Contract:" << endl;
+						m_test->printSource(cout, "    ", formatted);
+						m_test->printUpdatedSettings(cout, "    ", formatted);
+
+						cout << endl << outputMessages.str() << endl;
+						return result == TestCase::TestResult::FatalError ? Result::Exception : Result::Failure;
+				}
 			else
 			{
 				AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
@@ -187,23 +200,6 @@ TestTool::Result TestTool::process()
 			"Unknown exception during test." << endl;
 		return Result::Exception;
 	}
-
-	if (success)
-	{
-		AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
-		return Result::Success;
-	}
-	else
-	{
-		AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
-
-		AnsiColorized(cout, formatted, {BOLD, CYAN}) << "  Contract:" << endl;
-		m_test->printSource(cout, "    ", formatted);
-		m_test->printUpdatedSettings(cout, "    ", formatted);
-
-		cout << endl << outputMessages.str() << endl;
-		return Result::Failure;
-	}
 }
 
 TestTool::Request TestTool::handleResponse(bool _exception)