CommonSyntaxTest: Add support for syntax tests with custom expectations in addition to expected errors

test/CommonSyntaxTest.cpp

@@ -19,6 +19,7 @@
 #include <test/CommonSyntaxTest.h>
 #include <test/Common.h>
 #include <test/TestCase.h>
+#include <libsolutil/CommonIO.h>
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/test/unit_test.hpp>
@@ -29,6 +30,7 @@
 
 using namespace std;
 using namespace solidity;
+using namespace solidity::util;
 using namespace solidity::util::formatting;
 using namespace solidity::langutil;
 using namespace solidity::frontend;
@@ -66,6 +68,7 @@ CommonSyntaxTest::CommonSyntaxTest(string const& _filename, langutil::EVMVersion
 
 TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
+	parseCustomExpectations(m_reader.stream());
 	parseAndAnalyze();
 
 	return conclude(_stream, _linePrefix, _formatted);
@@ -73,7 +76,7 @@ TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _line
 
 TestCase::TestResult CommonSyntaxTest::conclude(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
-	if (m_expectations == m_errorList)
+	if (expectationsMatch())
 		return TestResult::Success;
 
 	printExpectationAndError(_stream, _linePrefix, _formatted);
@@ -84,9 +87,9 @@ void CommonSyntaxTest::printExpectationAndError(ostream& _stream, string const&
 {
 	string nextIndentLevel = _linePrefix + "  ";
 	util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
-	printErrorList(_stream, m_expectations, nextIndentLevel, _formatted);
+	printExpectedResult(_stream, nextIndentLevel, _formatted);
 	util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
-	printErrorList(_stream, m_errorList, nextIndentLevel, _formatted);
+	printObtainedResult(_stream, nextIndentLevel, _formatted);
 }
 
 void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const
@@ -149,6 +152,30 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix,
 		}
 }
 
+void CommonSyntaxTest::parseCustomExpectations(istream& _stream)
+{
+	string remainingExpectations = boost::trim_copy(readUntilEnd(_stream));
+	soltestAssert(
+		remainingExpectations.empty(),
+		"Found custom expectations not supported by the test case:\n" + remainingExpectations
+	);
+}
+
+bool CommonSyntaxTest::expectationsMatch()
+{
+	return m_expectations == m_errorList;
+}
+
+void CommonSyntaxTest::printExpectedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const
+{
+	printErrorList(_stream, m_expectations, _linePrefix, _formatted);
+}
+
+void CommonSyntaxTest::printObtainedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const
+{
+	printErrorList(_stream, m_errorList, _linePrefix, _formatted);
+}
+
 void CommonSyntaxTest::printErrorList(
 	ostream& _stream,
 	vector<SyntaxTestError> const& _errorList,
@@ -157,7 +184,10 @@ void CommonSyntaxTest::printErrorList(
 )
 {
 	if (_errorList.empty())
+	{
+		if (_formatted)
 			util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl;
+	}
 	else
 		for (auto const& error: _errorList)
 		{
@@ -194,12 +224,20 @@ string CommonSyntaxTest::errorMessage(util::Exception const& _e)
 
 vector<SyntaxTestError> CommonSyntaxTest::parseExpectations(istream& _stream)
 {
+	static string const customExpectationsDelimiter("// ----");
+
 	vector<SyntaxTestError> expectations;
 	string line;
 	while (getline(_stream, line))
 	{
 		auto it = line.begin();
 
+		// Anything below the delimiter is left up to the derived class to process in a custom way.
+		// The delimiter is optional and identical to the one that starts error expectations in
+		// TestCaseReader::parseSourcesAndSettingsWithLineNumber().
+		if (boost::algorithm::starts_with(line, customExpectationsDelimiter))
+			break;
+
 		skipSlashes(it, line.end());
 		skipWhitespace(it, line.end());
 

test/CommonSyntaxTest.h

@@ -62,14 +62,27 @@ public:
 	void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override
 	{
-		if (!m_errorList.empty())
+		printObtainedResult(_stream, _linePrefix, false);
-			printErrorList(_stream, m_errorList, _linePrefix, false);
 	}
 
 	static std::string errorMessage(util::Exception const& _e);
 protected:
+	/// Should be implemented by those derived test cases that want to allow extra expectations
+	/// after the error/warning expectations. The default implementation does not allow them and
+	/// fails instead.
+	/// @param _stream Input stream positioned at the beginning of the extra expectations.
+	virtual void parseCustomExpectations(std::istream& _stream);
+
 	virtual void parseAndAnalyze() = 0;
 
+	/// Should return true if obtained values match expectations.
+	/// The default implementation only compares the error list. Derived classes that support
+	/// custom expectations should override this to include them in the comparison.
+	virtual bool expectationsMatch();
+
+	virtual void printExpectedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const;
+	virtual void printObtainedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const;
+
 	static void printErrorList(
 		std::ostream& _stream,
 		std::vector<SyntaxTestError> const& _errors,

test/TestCaseReader.cpp

@@ -114,15 +114,17 @@ pair<SourceMap, size_t> TestCaseReader::parseSourcesAndSettingsWithLineNumber(is
 	static string const sourceDelimiterEnd("====");
 	static string const comment("// ");
 	static string const settingsDelimiter("// ====");
-	static string const delimiter("// ----");
+	static string const expectationsDelimiter("// ----");
 	bool sourcePart = true;
 	while (getline(_stream, line))
 	{
 		lineNumber++;
 
-		if (boost::algorithm::starts_with(line, delimiter))
+		// Anything below the delimiter is left up to the test case to process in a custom way.
+		if (boost::algorithm::starts_with(line, expectationsDelimiter))
 			break;
-		else if (boost::algorithm::starts_with(line, settingsDelimiter))
+
+		if (boost::algorithm::starts_with(line, settingsDelimiter))
 			sourcePart = false;
 		else if (sourcePart)
 		{