diff --git a/test/Common.cpp b/test/Common.cpp index 5c21744b2..acba2a016 100644 --- a/test/Common.cpp +++ b/test/Common.cpp @@ -75,6 +75,7 @@ CommonOptions::CommonOptions(std::string _caption): ) { options.add_options() + ("evm-version", po::value(&evmVersionString), "which evm version to use") ("testpath", po::value(&this->testPath)->default_value(dev::test::testPath()), "path to test files") ("ipcpath", po::value(&ipcPath)->default_value(IPCEnvOrDefaultPath()), "path to ipc socket") ("no-ipc", po::bool_switch(&disableIPC), "disable semantic tests") @@ -121,6 +122,20 @@ bool CommonOptions::parse(int argc, char const* const* argv) return true; } + +langutil::EVMVersion CommonOptions::evmVersion() const +{ + if (!evmVersionString.empty()) + { + auto version = langutil::EVMVersion::fromString(evmVersionString); + if (!version) + throw std::runtime_error("Invalid EVM version: " + evmVersionString); + return *version; + } + else + return langutil::EVMVersion(); +} + } } diff --git a/test/Common.h b/test/Common.h index 272db5a0f..f16646daa 100644 --- a/test/Common.h +++ b/test/Common.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -39,6 +40,8 @@ struct CommonOptions: boost::noncopyable bool disableIPC = false; bool disableSMT = false; + langutil::EVMVersion evmVersion() const; + virtual bool parse(int argc, char const* const* argv); // Throws a ConfigException on error virtual void validate() const; @@ -47,6 +50,9 @@ protected: CommonOptions(std::string caption = ""); boost::program_options::options_description options; + +private: + std::string evmVersionString; }; } diff --git a/test/ExecutionFramework.cpp b/test/ExecutionFramework.cpp index dcf61af66..6d38cab41 100644 --- a/test/ExecutionFramework.cpp +++ b/test/ExecutionFramework.cpp @@ -50,13 +50,13 @@ string getIPCSocketPath() } ExecutionFramework::ExecutionFramework(): - ExecutionFramework(getIPCSocketPath()) + ExecutionFramework(getIPCSocketPath(), dev::test::Options::get().evmVersion()) { } -ExecutionFramework::ExecutionFramework(string const& _ipcPath): +ExecutionFramework::ExecutionFramework(string const& _ipcPath, langutil::EVMVersion _evmVersion): m_rpc(RPCSession::instance(_ipcPath)), - m_evmVersion(dev::test::Options::get().evmVersion()), + m_evmVersion(_evmVersion), m_optimize(dev::test::Options::get().optimize), m_showMessages(dev::test::Options::get().showMessages), m_sender(m_rpc.account(0)) diff --git a/test/ExecutionFramework.h b/test/ExecutionFramework.h index f70c64b3f..27eb1da1f 100644 --- a/test/ExecutionFramework.h +++ b/test/ExecutionFramework.h @@ -53,7 +53,7 @@ class ExecutionFramework public: ExecutionFramework(); - explicit ExecutionFramework(std::string const& _ipcPath); + explicit ExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion _evmVersion); virtual ~ExecutionFramework() = default; virtual bytes const& compileAndRunWithoutCheck( diff --git a/test/Options.cpp b/test/Options.cpp index e7d4badb1..26a83d16f 100644 --- a/test/Options.cpp +++ b/test/Options.cpp @@ -53,22 +53,7 @@ Options::Options() options.add_options() ("optimize", po::bool_switch(&optimize), "enables optimization") ("abiencoderv2", po::bool_switch(&useABIEncoderV2), "enables abi encoder v2") - ("evm-version", po::value(&evmVersionString), "which evm version to use") ("show-messages", po::bool_switch(&showMessages), "enables message output"); parse(suite.argc, suite.argv); } - -langutil::EVMVersion Options::evmVersion() const -{ - if (!evmVersionString.empty()) - { - // We do this check as opposed to in the constructor because the BOOST_REQUIRE - // macros cannot yet be used in the constructor. - auto version = langutil::EVMVersion::fromString(evmVersionString); - BOOST_REQUIRE_MESSAGE(version, "Invalid EVM version: " + evmVersionString); - return *version; - } - else - return langutil::EVMVersion(); -} diff --git a/test/Options.h b/test/Options.h index 7f2ea430b..35a49c619 100644 --- a/test/Options.h +++ b/test/Options.h @@ -37,13 +37,9 @@ struct Options: CommonOptions bool showMessages = false; bool useABIEncoderV2 = false; - langutil::EVMVersion evmVersion() const; - static Options const& get(); private: - std::string evmVersionString; - Options(); }; diff --git a/test/TestCase.cpp b/test/TestCase.cpp index e9e2c9f21..6f3c92a53 100644 --- a/test/TestCase.cpp +++ b/test/TestCase.cpp @@ -17,6 +17,7 @@ #include +#include #include #include @@ -35,16 +36,57 @@ bool TestCase::isTestFilename(boost::filesystem::path const& _filename) !boost::starts_with(_filename.string(), "."); } +bool TestCase::supportedForEVMVersion(langutil::EVMVersion _evmVersion) const +{ + return boost::algorithm::none_of(m_evmVersionRules, [&](auto const& rule) { return !rule(_evmVersion); }); +} + string TestCase::parseSource(istream& _stream) { string source; string line; - string const delimiter("// ----"); + static string const delimiter("// ----"); + static string const evmVersion("// EVMVersion: "); + bool isTop = true; while (getline(_stream, line)) if (boost::algorithm::starts_with(line, delimiter)) break; else + { + if (isTop && boost::algorithm::starts_with(line, evmVersion)) + { + string versionString = line.substr(evmVersion.size() + 1); + auto version = langutil::EVMVersion::fromString(versionString); + if (!version) + throw runtime_error("Invalid EVM version: \"" + versionString + "\""); + switch (line.at(evmVersion.size())) + { + case '>': + m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) { + return version < _version; + }); + break; + case '<': + m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) { + return _version < version; + }); + break; + case '=': + m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) { + return _version == version; + }); + break; + case '!': + m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) { + return !(_version == version); + }); + break; + } + } + else + isTop = false; source += line + "\n"; + } return source; } diff --git a/test/TestCase.h b/test/TestCase.h index 52bca5274..86a109682 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -17,11 +17,15 @@ #pragma once +#include + #include +#include #include #include #include +#include namespace dev { @@ -46,6 +50,7 @@ public: { std::string filename; std::string ipcPath; + langutil::EVMVersion evmVersion; }; using TestCaseCreator = std::unique_ptr(*)(Config const&); @@ -69,8 +74,11 @@ public: static bool isTestFilename(boost::filesystem::path const& _filename); + /// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise. + bool supportedForEVMVersion(langutil::EVMVersion _evmVersion) const; + protected: - static std::string parseSource(std::istream& _file); + std::string parseSource(std::istream& _file); static void expect(std::string::iterator& _it, std::string::iterator _end, std::string::value_type _c); template @@ -86,7 +94,8 @@ protected: while (_it != _end && *_it == '/') ++_it; } - +private: + std::vector> m_evmVersionRules; }; } diff --git a/test/boostTest.cpp b/test/boostTest.cpp index 34412cb30..d9bf9c3fc 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -80,7 +80,7 @@ int registerTests( { int numTestsAdded = 0; fs::path fullpath = _basepath / _path; - TestCase::Config config{fullpath.string(), _ipcPath}; + TestCase::Config config{fullpath.string(), _ipcPath, dev::test::Options::get().evmVersion()}; if (fs::is_directory(fullpath)) { test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string()); @@ -104,8 +104,10 @@ int registerTests( try { stringstream errorStream; - if (!_testCaseCreator(config)->run(errorStream)) - BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str()); + auto testCase = _testCaseCreator(config); + if (testCase->supportedForEVMVersion(dev::test::Options::get().evmVersion())) + if (!testCase->run(errorStream)) + BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str()); } catch (boost::exception const& _e) { diff --git a/test/libsolidity/SMTCheckerJSONTest.cpp b/test/libsolidity/SMTCheckerJSONTest.cpp index 18a5bc410..d682c250e 100644 --- a/test/libsolidity/SMTCheckerJSONTest.cpp +++ b/test/libsolidity/SMTCheckerJSONTest.cpp @@ -35,8 +35,8 @@ using namespace dev; using namespace std; using namespace boost::unit_test; -SMTCheckerTest::SMTCheckerTest(string const& _filename) -: SyntaxTest(_filename) +SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _evmVersion) +: SyntaxTest(_filename, _evmVersion) { if (!boost::algorithm::ends_with(_filename, ".sol")) BOOST_THROW_EXCEPTION(runtime_error("Invalid test contract file name: \"" + _filename + "\".")); @@ -49,7 +49,7 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename) BOOST_THROW_EXCEPTION(runtime_error("Invalid JSON file.")); } -bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted) +bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { StandardCompiler compiler; diff --git a/test/libsolidity/SMTCheckerJSONTest.h b/test/libsolidity/SMTCheckerJSONTest.h index 256056689..5abbbff4d 100644 --- a/test/libsolidity/SMTCheckerJSONTest.h +++ b/test/libsolidity/SMTCheckerJSONTest.h @@ -35,11 +35,11 @@ class SMTCheckerTest: public SyntaxTest public: static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new SMTCheckerTest(_config.filename)); + return std::make_unique(_config.filename, _config.evmVersion); } - SMTCheckerTest(std::string const& _filename); + SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion); - bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; + bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; private: std::vector hashesFromJson(Json::Value const& _jsonObj, std::string const& _auxInput, std::string const& _smtlib); diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index f8c9fa110..1dcf91719 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -36,8 +36,8 @@ using namespace boost::unit_test; namespace fs = boost::filesystem; -SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath): - SolidityExecutionFramework(_ipcPath) +SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, langutil::EVMVersion _evmVersion): + SolidityExecutionFramework(_ipcPath, _evmVersion) { ifstream file(_filename); soltestAssert(file, "Cannot open test contract: \"" + _filename + "\"."); @@ -47,7 +47,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath): parseExpectations(file); } -bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted) +bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract."); @@ -87,7 +87,7 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _ return true; } -void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const +void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const { stringstream stream(m_source); string line; diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h index ccc8812a1..27e644220 100644 --- a/test/libsolidity/SemanticTest.h +++ b/test/libsolidity/SemanticTest.h @@ -44,12 +44,12 @@ class SemanticTest: public SolidityExecutionFramework, public TestCase { public: static std::unique_ptr create(Config const& _options) - { return std::make_unique(_options.filename, _options.ipcPath); } + { return std::make_unique(_options.filename, _options.ipcPath, _options.evmVersion); } - explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath); + explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion _evmVersion); - bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; - void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override; + bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; + void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override; /// Instantiates a test file parser that parses the additional comment section at the end of diff --git a/test/libsolidity/SolidityExecutionFramework.cpp b/test/libsolidity/SolidityExecutionFramework.cpp index 934c563f4..1b2322e89 100644 --- a/test/libsolidity/SolidityExecutionFramework.cpp +++ b/test/libsolidity/SolidityExecutionFramework.cpp @@ -33,7 +33,7 @@ SolidityExecutionFramework::SolidityExecutionFramework(): { } -SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath): - ExecutionFramework(_ipcPath) +SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion _evmVersion): + ExecutionFramework(_ipcPath, _evmVersion) { } diff --git a/test/libsolidity/SolidityExecutionFramework.h b/test/libsolidity/SolidityExecutionFramework.h index 37adbdf4e..44329280d 100644 --- a/test/libsolidity/SolidityExecutionFramework.h +++ b/test/libsolidity/SolidityExecutionFramework.h @@ -43,7 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework public: SolidityExecutionFramework(); - SolidityExecutionFramework(std::string const& _ipcPath); + SolidityExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion _evmVersion); virtual bytes const& compileAndRunWithoutCheck( std::string const& _sourceCode, diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp index f13b2e79c..d9900ab2b 100644 --- a/test/libsolidity/SyntaxTest.cpp +++ b/test/libsolidity/SyntaxTest.cpp @@ -52,7 +52,7 @@ int parseUnsignedInteger(string::iterator& _it, string::iterator _end) } -SyntaxTest::SyntaxTest(string const& _filename) +SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion): m_evmVersion(_evmVersion) { ifstream file(_filename); if (!file) @@ -63,12 +63,12 @@ SyntaxTest::SyntaxTest(string const& _filename) m_expectations = parseExpectations(file); } -bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted) +bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted) { string const versionPragma = "pragma solidity >=0.0;\n"; m_compiler.reset(); m_compiler.addSource("", versionPragma + m_source); - m_compiler.setEVMVersion(dev::test::Options::get().evmVersion()); + m_compiler.setEVMVersion(m_evmVersion); if (m_compiler.parse()) m_compiler.analyze(); @@ -95,7 +95,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _fo return printExpectationAndError(_stream, _linePrefix, _formatted); } -bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool const _formatted) +bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool _formatted) { if (m_expectations != m_errorList) { @@ -109,7 +109,7 @@ bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _lineP return true; } -void SyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool const _formatted) const +void SyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const { if (_formatted) { @@ -162,7 +162,7 @@ void SyntaxTest::printErrorList( ostream& _stream, vector const& _errorList, string const& _linePrefix, - bool const _formatted + bool _formatted ) { if (_errorList.empty()) diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index c331636ab..a7f5d0b98 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -54,12 +54,12 @@ class SyntaxTest: AnalysisFramework, public TestCase { public: static std::unique_ptr create(Config const& _config) - { return std::unique_ptr(new SyntaxTest(_config.filename)); } - SyntaxTest(std::string const& _filename); + { return std::make_unique(_config.filename, _config.evmVersion); } + SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion); - bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; + bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override; - void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const override; + void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override; void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override { if (!m_errorList.empty()) @@ -72,16 +72,17 @@ protected: std::ostream& _stream, std::vector const& _errors, std::string const& _linePrefix, - bool const _formatted = false + bool _formatted = false ); - virtual bool printExpectationAndError(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false); + virtual bool printExpectationAndError(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false); static std::vector parseExpectations(std::istream& _stream); std::string m_source; std::vector m_expectations; std::vector m_errorList; + langutil::EVMVersion const m_evmVersion; }; } diff --git a/test/tools/IsolTestOptions.h b/test/tools/IsolTestOptions.h index a13ff2a25..95bb5cc92 100644 --- a/test/tools/IsolTestOptions.h +++ b/test/tools/IsolTestOptions.h @@ -19,6 +19,8 @@ #pragma once +#include + #include namespace dev diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index 9c212346c..1269f5e80 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -47,13 +47,15 @@ namespace fs = boost::filesystem; struct TestStats { - int successCount; - int testCount; - operator bool() const { return successCount == testCount; } + int successCount = 0; + int testCount = 0; + int skippedCount = 0; + operator bool() const noexcept { return successCount + skippedCount == testCount; } TestStats& operator+=(TestStats const& _other) noexcept { successCount += _other.successCount; testCount += _other.testCount; + skippedCount += _other.skippedCount; return *this; } }; @@ -66,15 +68,17 @@ public: string const& _name, fs::path const& _path, string const& _ipcPath, - bool _formatted - ): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted) + bool _formatted, + langutil::EVMVersion _evmVersion + ): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion) {} enum class Result { Success, Failure, - Exception + Exception, + Skipped }; Result process(); @@ -84,7 +88,8 @@ public: fs::path const& _basepath, fs::path const& _path, string const& _ipcPath, - bool const _formatted + bool _formatted, + langutil::EVMVersion _evmVersion ); static string editor; @@ -96,13 +101,14 @@ private: Quit }; - Request handleResponse(bool const _exception); + Request handleResponse(bool _exception); TestCase::TestCaseCreator m_testCaseCreator; string const m_name; fs::path const m_path; string m_ipcPath; bool const m_formatted = false; + langutil::EVMVersion const m_evmVersion; unique_ptr m_test; static bool m_exitRequested; }; @@ -119,8 +125,14 @@ TestTool::Result TestTool::process() try { - m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath}); - success = m_test->run(outputMessages, " ", m_formatted); + m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion}); + if (m_test->supportedForEVMVersion(m_evmVersion)) + success = m_test->run(outputMessages, " ", m_formatted); + else + { + AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl; + return Result::Skipped; + } } catch(boost::exception const& _e) { @@ -158,7 +170,7 @@ TestTool::Result TestTool::process() } } -TestTool::Request TestTool::handleResponse(bool const _exception) +TestTool::Request TestTool::handleResponse(bool _exception) { if (_exception) cout << "(e)dit/(s)kip/(q)uit? "; @@ -204,13 +216,15 @@ TestStats TestTool::processPath( fs::path const& _basepath, fs::path const& _path, string const& _ipcPath, - bool const _formatted + bool _formatted, + langutil::EVMVersion _evmVersion ) { std::queue paths; paths.push(_path); int successCount = 0; int testCount = 0; + int skippedCount = 0; while (!paths.empty()) { @@ -235,7 +249,7 @@ TestStats TestTool::processPath( else { ++testCount; - TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted); + TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion); auto result = testTool.process(); switch(result) @@ -254,6 +268,7 @@ TestStats TestTool::processPath( break; case Request::Skip: paths.pop(); + ++skippedCount; break; } break; @@ -261,11 +276,15 @@ TestStats TestTool::processPath( paths.pop(); ++successCount; break; + case Result::Skipped: + paths.pop(); + ++skippedCount; + break; } } } - return { successCount, testCount }; + return { successCount, testCount, skippedCount }; } @@ -298,7 +317,8 @@ boost::optional runTestSuite( fs::path const& _subdirectory, string const& _ipcPath, TestCase::TestCaseCreator _testCaseCreator, - bool _formatted + bool _formatted, + langutil::EVMVersion _evmVersion ) { fs::path testPath = _basePath / _subdirectory; @@ -309,14 +329,21 @@ boost::optional runTestSuite( return {}; } - TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted); + TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion); cout << endl << _name << " Test Summary: "; AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) << stats.successCount << "/" << stats.testCount; - cout << " tests successful." << endl << endl; + cout << " tests successful"; + if (stats.skippedCount > 0) + { + cout << " ("; + AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount; + cout<< " tests skipped)"; + } + cout << "." << endl << endl; return stats; } @@ -354,7 +381,7 @@ int main(int argc, char const *argv[]) if (ts.smt && options.disableSMT) continue; - if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor)) + if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion())) global_stats += *stats; else return 1; @@ -363,7 +390,14 @@ int main(int argc, char const *argv[]) cout << endl << "Summary: "; AnsiColorized(cout, !options.noColor, {BOLD, global_stats ? GREEN : RED}) << global_stats.successCount << "/" << global_stats.testCount; - cout << " tests successful." << endl; + cout << " tests successful"; + if (global_stats.skippedCount > 0) + { + cout << " ("; + AnsiColorized(cout, !options.noColor, {BOLD, YELLOW}) << global_stats.skippedCount; + cout << " tests skipped)"; + } + cout << "." << endl; return global_stats ? 0 : 1; }