From c5b81b66cd06e49b14e29606247ec2fd5ad4d441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:26:01 +0200 Subject: [PATCH 1/5] Define Error::severity() --- liblangutil/Exceptions.h | 1 + 1 file changed, 1 insertion(+) diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h index fc7286ed1..df3128fe2 100644 --- a/liblangutil/Exceptions.h +++ b/liblangutil/Exceptions.h @@ -206,6 +206,7 @@ public: ErrorId errorId() const { return m_errorId; } Type type() const { return m_type; } + Severity severity() const { return errorSeverity(m_type); } SourceLocation const* sourceLocation() const noexcept; SecondarySourceLocation const* secondarySourceLocation() const noexcept; From b1ead4af941f337c8103abcb6868777fef7aca02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:22:23 +0200 Subject: [PATCH 2/5] Order Error::Severity enum from most to least severe - Also reorder Error::Type to match initial values for some consistency --- liblangutil/Exceptions.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h index df3128fe2..65362df31 100644 --- a/liblangutil/Exceptions.h +++ b/liblangutil/Exceptions.h @@ -170,6 +170,8 @@ class Error: virtual public util::Exception public: enum class Type { + Info, + Warning, CodeGenerationError, DeclarationError, DocstringParsingError, @@ -185,15 +187,14 @@ public: UnimplementedFeatureError, YulException, SMTLogicException, - Warning, - Info }; enum class Severity { - Error, + // NOTE: We rely on these being ordered from least to most severe. + Info, Warning, - Info + Error, }; Error( From 73b9077ab08860ccbe57f2fcf474446b723d636f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:13:43 +0200 Subject: [PATCH 3/5] SyntaxTest: Default-initialize boolean fields without showing the value - These get re-initialized in constructor anyway. The only purpose if initializing here is our convention to always initialize primitive types at declaration time. We don't want to have to repeat the defaults though. --- test/libsolidity/SyntaxTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index f4ca6d9bc..f9fd65319 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -48,7 +48,7 @@ protected: void parseAndAnalyze() override; virtual void filterObtainedErrors(); - bool m_optimiseYul = true; + bool m_optimiseYul{}; }; } From c965d6332c88e626249c7c3086b1f1c5df6129da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 21 Aug 2023 10:29:16 +0200 Subject: [PATCH 4/5] SyntaxTest: Allow derived test cases to filter out warnings and infos --- test/libsolidity/SyntaxTest.cpp | 11 ++++++++++- test/libsolidity/SyntaxTest.h | 7 ++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp index e0d7a17e9..ad53c2b26 100644 --- a/test/libsolidity/SyntaxTest.cpp +++ b/test/libsolidity/SyntaxTest.cpp @@ -38,7 +38,13 @@ using namespace solidity::frontend::test; using namespace boost::unit_test; namespace fs = boost::filesystem; -SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion): CommonSyntaxTest(_filename, _evmVersion) +SyntaxTest::SyntaxTest( + string const& _filename, + langutil::EVMVersion _evmVersion, + Error::Severity _minSeverity +): + CommonSyntaxTest(_filename, _evmVersion), + m_minSeverity(_minSeverity) { m_optimiseYul = m_reader.boolSetting("optimize-yul", true); } @@ -98,6 +104,9 @@ void SyntaxTest::filterObtainedErrors() { for (auto const& currentError: filteredErrors()) { + if (currentError->severity() < m_minSeverity) + continue; + int locationStart = -1; int locationEnd = -1; string sourceName; diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index f9fd65319..a85e1f53f 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -41,7 +41,11 @@ public: { return std::make_unique(_config.filename, _config.evmVersion); } - SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion); + SyntaxTest( + std::string const& _filename, + langutil::EVMVersion _evmVersion, + langutil::Error::Severity _minSeverity = langutil::Error::Severity::Info + ); protected: virtual void setupCompiler(); @@ -49,6 +53,7 @@ protected: virtual void filterObtainedErrors(); bool m_optimiseYul{}; + langutil::Error::Severity m_minSeverity{}; }; } From e847596e3950862a23e88cb21df4062ad0f3a398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 18 Aug 2023 15:44:13 +0200 Subject: [PATCH 5/5] CommonSyntaxTest: Add support for syntax tests with custom expectations in addition to expected errors --- test/CommonSyntaxTest.cpp | 46 +++++++++++++++++++++++++++++++++++---- test/CommonSyntaxTest.h | 17 +++++++++++++-- test/TestCaseReader.cpp | 8 ++++--- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/test/CommonSyntaxTest.cpp b/test/CommonSyntaxTest.cpp index f794439de..79b597804 100644 --- a/test/CommonSyntaxTest.cpp +++ b/test/CommonSyntaxTest.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ using namespace std; using namespace solidity; +using namespace solidity::util; using namespace solidity::util::formatting; using namespace solidity::langutil; using namespace solidity::frontend; @@ -66,6 +68,7 @@ CommonSyntaxTest::CommonSyntaxTest(string const& _filename, langutil::EVMVersion TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { + parseCustomExpectations(m_reader.stream()); parseAndAnalyze(); return conclude(_stream, _linePrefix, _formatted); @@ -73,7 +76,7 @@ TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _line TestCase::TestResult CommonSyntaxTest::conclude(ostream& _stream, string const& _linePrefix, bool _formatted) { - if (m_expectations == m_errorList) + if (expectationsMatch()) return TestResult::Success; printExpectationAndError(_stream, _linePrefix, _formatted); @@ -84,9 +87,9 @@ void CommonSyntaxTest::printExpectationAndError(ostream& _stream, string const& { string nextIndentLevel = _linePrefix + " "; util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; - printErrorList(_stream, m_expectations, nextIndentLevel, _formatted); + printExpectedResult(_stream, nextIndentLevel, _formatted); util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; - printErrorList(_stream, m_errorList, nextIndentLevel, _formatted); + printObtainedResult(_stream, nextIndentLevel, _formatted); } void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const @@ -149,6 +152,30 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, } } +void CommonSyntaxTest::parseCustomExpectations(istream& _stream) +{ + string remainingExpectations = boost::trim_copy(readUntilEnd(_stream)); + soltestAssert( + remainingExpectations.empty(), + "Found custom expectations not supported by the test case:\n" + remainingExpectations + ); +} + +bool CommonSyntaxTest::expectationsMatch() +{ + return m_expectations == m_errorList; +} + +void CommonSyntaxTest::printExpectedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_expectations, _linePrefix, _formatted); +} + +void CommonSyntaxTest::printObtainedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_errorList, _linePrefix, _formatted); +} + void CommonSyntaxTest::printErrorList( ostream& _stream, vector const& _errorList, @@ -157,7 +184,10 @@ void CommonSyntaxTest::printErrorList( ) { if (_errorList.empty()) - util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + { + if (_formatted) + util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + } else for (auto const& error: _errorList) { @@ -194,12 +224,20 @@ string CommonSyntaxTest::errorMessage(util::Exception const& _e) vector CommonSyntaxTest::parseExpectations(istream& _stream) { + static string const customExpectationsDelimiter("// ----"); + vector expectations; string line; while (getline(_stream, line)) { auto it = line.begin(); + // Anything below the delimiter is left up to the derived class to process in a custom way. + // The delimiter is optional and identical to the one that starts error expectations in + // TestCaseReader::parseSourcesAndSettingsWithLineNumber(). + if (boost::algorithm::starts_with(line, customExpectationsDelimiter)) + break; + skipSlashes(it, line.end()); skipWhitespace(it, line.end()); diff --git a/test/CommonSyntaxTest.h b/test/CommonSyntaxTest.h index 169824252..606f61a88 100644 --- a/test/CommonSyntaxTest.h +++ b/test/CommonSyntaxTest.h @@ -62,14 +62,27 @@ public: void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override { - if (!m_errorList.empty()) - printErrorList(_stream, m_errorList, _linePrefix, false); + printObtainedResult(_stream, _linePrefix, false); } static std::string errorMessage(util::Exception const& _e); protected: + /// Should be implemented by those derived test cases that want to allow extra expectations + /// after the error/warning expectations. The default implementation does not allow them and + /// fails instead. + /// @param _stream Input stream positioned at the beginning of the extra expectations. + virtual void parseCustomExpectations(std::istream& _stream); + virtual void parseAndAnalyze() = 0; + /// Should return true if obtained values match expectations. + /// The default implementation only compares the error list. Derived classes that support + /// custom expectations should override this to include them in the comparison. + virtual bool expectationsMatch(); + + virtual void printExpectedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + virtual void printObtainedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + static void printErrorList( std::ostream& _stream, std::vector const& _errors, diff --git a/test/TestCaseReader.cpp b/test/TestCaseReader.cpp index 43e4a6fc3..7ad752ef8 100644 --- a/test/TestCaseReader.cpp +++ b/test/TestCaseReader.cpp @@ -114,15 +114,17 @@ pair TestCaseReader::parseSourcesAndSettingsWithLineNumber(is static string const sourceDelimiterEnd("===="); static string const comment("// "); static string const settingsDelimiter("// ===="); - static string const delimiter("// ----"); + static string const expectationsDelimiter("// ----"); bool sourcePart = true; while (getline(_stream, line)) { lineNumber++; - if (boost::algorithm::starts_with(line, delimiter)) + // Anything below the delimiter is left up to the test case to process in a custom way. + if (boost::algorithm::starts_with(line, expectationsDelimiter)) break; - else if (boost::algorithm::starts_with(line, settingsDelimiter)) + + if (boost::algorithm::starts_with(line, settingsDelimiter)) sourcePart = false; else if (sourcePart) {