From e847596e3950862a23e88cb21df4062ad0f3a398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 18 Aug 2023 15:44:13 +0200 Subject: [PATCH] CommonSyntaxTest: Add support for syntax tests with custom expectations in addition to expected errors --- test/CommonSyntaxTest.cpp | 46 +++++++++++++++++++++++++++++++++++---- test/CommonSyntaxTest.h | 17 +++++++++++++-- test/TestCaseReader.cpp | 8 ++++--- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/test/CommonSyntaxTest.cpp b/test/CommonSyntaxTest.cpp index f794439de..79b597804 100644 --- a/test/CommonSyntaxTest.cpp +++ b/test/CommonSyntaxTest.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ using namespace std; using namespace solidity; +using namespace solidity::util; using namespace solidity::util::formatting; using namespace solidity::langutil; using namespace solidity::frontend; @@ -66,6 +68,7 @@ CommonSyntaxTest::CommonSyntaxTest(string const& _filename, langutil::EVMVersion TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { + parseCustomExpectations(m_reader.stream()); parseAndAnalyze(); return conclude(_stream, _linePrefix, _formatted); @@ -73,7 +76,7 @@ TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _line TestCase::TestResult CommonSyntaxTest::conclude(ostream& _stream, string const& _linePrefix, bool _formatted) { - if (m_expectations == m_errorList) + if (expectationsMatch()) return TestResult::Success; printExpectationAndError(_stream, _linePrefix, _formatted); @@ -84,9 +87,9 @@ void CommonSyntaxTest::printExpectationAndError(ostream& _stream, string const& { string nextIndentLevel = _linePrefix + " "; util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; - printErrorList(_stream, m_expectations, nextIndentLevel, _formatted); + printExpectedResult(_stream, nextIndentLevel, _formatted); util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; - printErrorList(_stream, m_errorList, nextIndentLevel, _formatted); + printObtainedResult(_stream, nextIndentLevel, _formatted); } void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const @@ -149,6 +152,30 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, } } +void CommonSyntaxTest::parseCustomExpectations(istream& _stream) +{ + string remainingExpectations = boost::trim_copy(readUntilEnd(_stream)); + soltestAssert( + remainingExpectations.empty(), + "Found custom expectations not supported by the test case:\n" + remainingExpectations + ); +} + +bool CommonSyntaxTest::expectationsMatch() +{ + return m_expectations == m_errorList; +} + +void CommonSyntaxTest::printExpectedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_expectations, _linePrefix, _formatted); +} + +void CommonSyntaxTest::printObtainedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_errorList, _linePrefix, _formatted); +} + void CommonSyntaxTest::printErrorList( ostream& _stream, vector const& _errorList, @@ -157,7 +184,10 @@ void CommonSyntaxTest::printErrorList( ) { if (_errorList.empty()) - util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + { + if (_formatted) + util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + } else for (auto const& error: _errorList) { @@ -194,12 +224,20 @@ string CommonSyntaxTest::errorMessage(util::Exception const& _e) vector CommonSyntaxTest::parseExpectations(istream& _stream) { + static string const customExpectationsDelimiter("// ----"); + vector expectations; string line; while (getline(_stream, line)) { auto it = line.begin(); + // Anything below the delimiter is left up to the derived class to process in a custom way. + // The delimiter is optional and identical to the one that starts error expectations in + // TestCaseReader::parseSourcesAndSettingsWithLineNumber(). + if (boost::algorithm::starts_with(line, customExpectationsDelimiter)) + break; + skipSlashes(it, line.end()); skipWhitespace(it, line.end()); diff --git a/test/CommonSyntaxTest.h b/test/CommonSyntaxTest.h index 169824252..606f61a88 100644 --- a/test/CommonSyntaxTest.h +++ b/test/CommonSyntaxTest.h @@ -62,14 +62,27 @@ public: void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override { - if (!m_errorList.empty()) - printErrorList(_stream, m_errorList, _linePrefix, false); + printObtainedResult(_stream, _linePrefix, false); } static std::string errorMessage(util::Exception const& _e); protected: + /// Should be implemented by those derived test cases that want to allow extra expectations + /// after the error/warning expectations. The default implementation does not allow them and + /// fails instead. + /// @param _stream Input stream positioned at the beginning of the extra expectations. + virtual void parseCustomExpectations(std::istream& _stream); + virtual void parseAndAnalyze() = 0; + /// Should return true if obtained values match expectations. + /// The default implementation only compares the error list. Derived classes that support + /// custom expectations should override this to include them in the comparison. + virtual bool expectationsMatch(); + + virtual void printExpectedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + virtual void printObtainedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + static void printErrorList( std::ostream& _stream, std::vector const& _errors, diff --git a/test/TestCaseReader.cpp b/test/TestCaseReader.cpp index 43e4a6fc3..7ad752ef8 100644 --- a/test/TestCaseReader.cpp +++ b/test/TestCaseReader.cpp @@ -114,15 +114,17 @@ pair TestCaseReader::parseSourcesAndSettingsWithLineNumber(is static string const sourceDelimiterEnd("===="); static string const comment("// "); static string const settingsDelimiter("// ===="); - static string const delimiter("// ----"); + static string const expectationsDelimiter("// ----"); bool sourcePart = true; while (getline(_stream, line)) { lineNumber++; - if (boost::algorithm::starts_with(line, delimiter)) + // Anything below the delimiter is left up to the test case to process in a custom way. + if (boost::algorithm::starts_with(line, expectationsDelimiter)) break; - else if (boost::algorithm::starts_with(line, settingsDelimiter)) + + if (boost::algorithm::starts_with(line, settingsDelimiter)) sourcePart = false; else if (sourcePart) {