diff --git a/liblangutil/Exceptions.h b/liblangutil/Exceptions.h index fc7286ed1..65362df31 100644 --- a/liblangutil/Exceptions.h +++ b/liblangutil/Exceptions.h @@ -170,6 +170,8 @@ class Error: virtual public util::Exception public: enum class Type { + Info, + Warning, CodeGenerationError, DeclarationError, DocstringParsingError, @@ -185,15 +187,14 @@ public: UnimplementedFeatureError, YulException, SMTLogicException, - Warning, - Info }; enum class Severity { - Error, + // NOTE: We rely on these being ordered from least to most severe. + Info, Warning, - Info + Error, }; Error( @@ -206,6 +207,7 @@ public: ErrorId errorId() const { return m_errorId; } Type type() const { return m_type; } + Severity severity() const { return errorSeverity(m_type); } SourceLocation const* sourceLocation() const noexcept; SecondarySourceLocation const* secondarySourceLocation() const noexcept; diff --git a/test/CommonSyntaxTest.cpp b/test/CommonSyntaxTest.cpp index f794439de..79b597804 100644 --- a/test/CommonSyntaxTest.cpp +++ b/test/CommonSyntaxTest.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ using namespace std; using namespace solidity; +using namespace solidity::util; using namespace solidity::util::formatting; using namespace solidity::langutil; using namespace solidity::frontend; @@ -66,6 +68,7 @@ CommonSyntaxTest::CommonSyntaxTest(string const& _filename, langutil::EVMVersion TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { + parseCustomExpectations(m_reader.stream()); parseAndAnalyze(); return conclude(_stream, _linePrefix, _formatted); @@ -73,7 +76,7 @@ TestCase::TestResult CommonSyntaxTest::run(ostream& _stream, string const& _line TestCase::TestResult CommonSyntaxTest::conclude(ostream& _stream, string const& _linePrefix, bool _formatted) { - if (m_expectations == m_errorList) + if (expectationsMatch()) return TestResult::Success; printExpectationAndError(_stream, _linePrefix, _formatted); @@ -84,9 +87,9 @@ void CommonSyntaxTest::printExpectationAndError(ostream& _stream, string const& { string nextIndentLevel = _linePrefix + " "; util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; - printErrorList(_stream, m_expectations, nextIndentLevel, _formatted); + printExpectedResult(_stream, nextIndentLevel, _formatted); util::AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; - printErrorList(_stream, m_errorList, nextIndentLevel, _formatted); + printObtainedResult(_stream, nextIndentLevel, _formatted); } void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const @@ -149,6 +152,30 @@ void CommonSyntaxTest::printSource(ostream& _stream, string const& _linePrefix, } } +void CommonSyntaxTest::parseCustomExpectations(istream& _stream) +{ + string remainingExpectations = boost::trim_copy(readUntilEnd(_stream)); + soltestAssert( + remainingExpectations.empty(), + "Found custom expectations not supported by the test case:\n" + remainingExpectations + ); +} + +bool CommonSyntaxTest::expectationsMatch() +{ + return m_expectations == m_errorList; +} + +void CommonSyntaxTest::printExpectedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_expectations, _linePrefix, _formatted); +} + +void CommonSyntaxTest::printObtainedResult(ostream& _stream, string const& _linePrefix, bool _formatted) const +{ + printErrorList(_stream, m_errorList, _linePrefix, _formatted); +} + void CommonSyntaxTest::printErrorList( ostream& _stream, vector const& _errorList, @@ -157,7 +184,10 @@ void CommonSyntaxTest::printErrorList( ) { if (_errorList.empty()) - util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + { + if (_formatted) + util::AnsiColorized(_stream, _formatted, {BOLD, GREEN}) << _linePrefix << "Success" << endl; + } else for (auto const& error: _errorList) { @@ -194,12 +224,20 @@ string CommonSyntaxTest::errorMessage(util::Exception const& _e) vector CommonSyntaxTest::parseExpectations(istream& _stream) { + static string const customExpectationsDelimiter("// ----"); + vector expectations; string line; while (getline(_stream, line)) { auto it = line.begin(); + // Anything below the delimiter is left up to the derived class to process in a custom way. + // The delimiter is optional and identical to the one that starts error expectations in + // TestCaseReader::parseSourcesAndSettingsWithLineNumber(). + if (boost::algorithm::starts_with(line, customExpectationsDelimiter)) + break; + skipSlashes(it, line.end()); skipWhitespace(it, line.end()); diff --git a/test/CommonSyntaxTest.h b/test/CommonSyntaxTest.h index 169824252..606f61a88 100644 --- a/test/CommonSyntaxTest.h +++ b/test/CommonSyntaxTest.h @@ -62,14 +62,27 @@ public: void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override { - if (!m_errorList.empty()) - printErrorList(_stream, m_errorList, _linePrefix, false); + printObtainedResult(_stream, _linePrefix, false); } static std::string errorMessage(util::Exception const& _e); protected: + /// Should be implemented by those derived test cases that want to allow extra expectations + /// after the error/warning expectations. The default implementation does not allow them and + /// fails instead. + /// @param _stream Input stream positioned at the beginning of the extra expectations. + virtual void parseCustomExpectations(std::istream& _stream); + virtual void parseAndAnalyze() = 0; + /// Should return true if obtained values match expectations. + /// The default implementation only compares the error list. Derived classes that support + /// custom expectations should override this to include them in the comparison. + virtual bool expectationsMatch(); + + virtual void printExpectedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + virtual void printObtainedResult(std::ostream& _stream, std::string const& _linePrefix, bool _formatted) const; + static void printErrorList( std::ostream& _stream, std::vector const& _errors, diff --git a/test/TestCaseReader.cpp b/test/TestCaseReader.cpp index 43e4a6fc3..7ad752ef8 100644 --- a/test/TestCaseReader.cpp +++ b/test/TestCaseReader.cpp @@ -114,15 +114,17 @@ pair TestCaseReader::parseSourcesAndSettingsWithLineNumber(is static string const sourceDelimiterEnd("===="); static string const comment("// "); static string const settingsDelimiter("// ===="); - static string const delimiter("// ----"); + static string const expectationsDelimiter("// ----"); bool sourcePart = true; while (getline(_stream, line)) { lineNumber++; - if (boost::algorithm::starts_with(line, delimiter)) + // Anything below the delimiter is left up to the test case to process in a custom way. + if (boost::algorithm::starts_with(line, expectationsDelimiter)) break; - else if (boost::algorithm::starts_with(line, settingsDelimiter)) + + if (boost::algorithm::starts_with(line, settingsDelimiter)) sourcePart = false; else if (sourcePart) { diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp index e0d7a17e9..ad53c2b26 100644 --- a/test/libsolidity/SyntaxTest.cpp +++ b/test/libsolidity/SyntaxTest.cpp @@ -38,7 +38,13 @@ using namespace solidity::frontend::test; using namespace boost::unit_test; namespace fs = boost::filesystem; -SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion): CommonSyntaxTest(_filename, _evmVersion) +SyntaxTest::SyntaxTest( + string const& _filename, + langutil::EVMVersion _evmVersion, + Error::Severity _minSeverity +): + CommonSyntaxTest(_filename, _evmVersion), + m_minSeverity(_minSeverity) { m_optimiseYul = m_reader.boolSetting("optimize-yul", true); } @@ -98,6 +104,9 @@ void SyntaxTest::filterObtainedErrors() { for (auto const& currentError: filteredErrors()) { + if (currentError->severity() < m_minSeverity) + continue; + int locationStart = -1; int locationEnd = -1; string sourceName; diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index f4ca6d9bc..a85e1f53f 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -41,14 +41,19 @@ public: { return std::make_unique(_config.filename, _config.evmVersion); } - SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion); + SyntaxTest( + std::string const& _filename, + langutil::EVMVersion _evmVersion, + langutil::Error::Severity _minSeverity = langutil::Error::Severity::Info + ); protected: virtual void setupCompiler(); void parseAndAnalyze() override; virtual void filterObtainedErrors(); - bool m_optimiseYul = true; + bool m_optimiseYul{}; + langutil::Error::Severity m_minSeverity{}; }; }