Introduced TestCase::shouldRun().

test/TestCase.cpp

@@ -15,6 +15,7 @@
 	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <test/Common.h>
 #include <test/TestCase.h>
 
 #include <libsolutil/StringUtils.h>
@@ -52,14 +53,18 @@ bool TestCase::isTestFilename(boost::filesystem::path const& _filename)
 			!boost::starts_with(_filename.string(), ".");
 }
 
-bool TestCase::validateSettings(langutil::EVMVersion)
+void TestCase::validateSettings()
 {
 	if (!m_settings.empty())
 		throw runtime_error(
 			"Unknown setting(s): " +
 			util::joinHumanReadable(m_settings | boost::adaptors::map_keys)
 		);
-	return true;
+}
+
+bool TestCase::shouldRun()
+{
+	return m_shouldRun;
 }
 
 pair<map<string, string>, size_t> TestCase::parseSourcesAndSettingsWithLineNumbers(istream& _stream)
@@ -157,20 +162,19 @@ void TestCase::expect(string::iterator& _it, string::iterator _end, string::valu
 	++_it;
 }
 
-bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVersion)
+void EVMVersionRestrictedTestCase::validateSettings()
 {
 	if (!m_settings.count("EVMVersion"))
-		return true;
+		return;
 
 	string versionString = m_settings["EVMVersion"];
 	m_validatedSettings["EVMVersion"] = versionString;
 	m_settings.erase("EVMVersion");
 
-	if (!TestCase::validateSettings(_evmVersion))
+	TestCase::validateSettings();
-		return false;
 
 	if (versionString.empty())
-		return true;
+		return;
 
 	string comparator;
 	size_t versionBegin = 0;
@@ -188,18 +192,23 @@ bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVer
 	if (!version)
 		BOOST_THROW_EXCEPTION(runtime_error{"Invalid EVM version: \"" + versionString + "\""});
 
+	langutil::EVMVersion evmVersion = solidity::test::CommonOptions::get().evmVersion();
+	bool comparisonResult;
 	if (comparator == ">")
-		return _evmVersion > version;
+		comparisonResult = evmVersion > version;
 	else if (comparator == ">=")
-		return _evmVersion >= version;
+		comparisonResult = evmVersion >= version;
 	else if (comparator == "<")
-		return _evmVersion < version;
+		comparisonResult = evmVersion < version;
 	else if (comparator == "<=")
-		return _evmVersion <= version;
+		comparisonResult = evmVersion <= version;
 	else if (comparator == "=")
-		return _evmVersion == version;
+		comparisonResult = evmVersion == version;
 	else if (comparator == "!")
-		return !(_evmVersion == version);
+		comparisonResult = !(evmVersion == version);
 	else
 		BOOST_THROW_EXCEPTION(runtime_error{"Invalid EVM comparator: \"" + comparator + "\""});
+
+	if (!comparisonResult)
+		m_shouldRun = false;
 }

test/TestCase.h

@@ -70,10 +70,12 @@ public:
 
 	/// Validates the settings, i.e. moves them from m_settings to m_validatedSettings.
 	/// Throws a runtime exception if any setting is left at this class (i.e. unknown setting).
+	virtual void validateSettings();
+
 	/// Returns true, if the test case is supported in the current environment and false
 	/// otherwise which causes this test to be skipped.
 	/// This might check e.g. for restrictions on the EVM version.
-	virtual bool validateSettings(langutil::EVMVersion /*_evmVersion*/);
+	bool shouldRun();
 
 protected:
 	std::pair<std::map<std::string, std::string>, std::size_t> parseSourcesAndSettingsWithLineNumbers(std::istream& _file);
@@ -102,13 +104,14 @@ protected:
 	std::map<std::string, std::string> m_settings;
 	/// Updated settings after validation.
 	std::map<std::string, std::string> m_validatedSettings;
+
+	bool m_shouldRun = true;
 };
 
 class EVMVersionRestrictedTestCase: public TestCase
 {
 public:
-	/// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise.
+	void validateSettings() override;
-	bool validateSettings(langutil::EVMVersion _evmVersion) override;
 };
 
 }

test/boostTest.cpp

@@ -94,7 +94,8 @@ int registerTests(
 					{
 						stringstream errorStream;
 						auto testCase = _testCaseCreator(config);
-						if (testCase->validateSettings(solidity::test::CommonOptions::get().evmVersion()))
+						testCase->validateSettings();
+						if (testCase->shouldRun())
 							switch (testCase->run(errorStream))
 							{
 								case TestCase::TestResult::Success:

test/libsolidity/SMTCheckerTest.cpp

@@ -44,6 +44,15 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev
 	}
 	else
 		m_enabledSolvers = smt::SMTSolverChoice::All();
+
+	auto available = ModelChecker::availableSolvers();
+	if (!available.z3)
+		m_enabledSolvers.z3 = false;
+	if (!available.cvc4)
+		m_enabledSolvers.cvc4 = false;
+
+	if (m_enabledSolvers.none())
+		m_shouldRun = false;
 }
 
 TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
@@ -55,17 +64,3 @@ TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePr
 
 	return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
 }
-
-bool SMTCheckerTest::validateSettings(langutil::EVMVersion _evmVersion)
-{
-	auto available = ModelChecker::availableSolvers();
-	if (!available.z3)
-		m_enabledSolvers.z3 = false;
-	if (!available.cvc4)
-		m_enabledSolvers.cvc4 = false;
-
-	if (m_enabledSolvers.none())
-		return false;
-
-	return SyntaxTest::validateSettings(_evmVersion);
-}

test/libsolidity/SMTCheckerTest.h

@@ -37,8 +37,6 @@ public:
 
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 
-	bool validateSettings(langutil::EVMVersion _evmVersion) override;
-
 protected:
 	/// This is set via option SMTSolvers in the test.
 	/// The possible options are `all`, `z3`, `cvc4`, `none`,

test/libsolidity/SemanticTest.cpp

@@ -71,6 +71,9 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer
 		m_settings.erase("ABIEncoderV1Only");
 	}
 
+	if (m_runWithABIEncoderV1Only && solidity::test::CommonOptions::get().useABIEncoderV2)
+		m_shouldRun = false;
+
 	if (m_settings.count("revertStrings"))
 	{
 		auto revertStrings = revertStringsFromString(m_settings["revertStrings"]);
@@ -90,13 +93,6 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer
 	soltestAssert(!m_tests.empty(), "No tests specified in " + _filename);
 }
 
-bool SemanticTest::validateSettings(langutil::EVMVersion _evmVersion)
-{
-	if (m_runWithABIEncoderV1Only && solidity::test::CommonOptions::get().useABIEncoderV2)
-		return false;
-	return EVMVersionRestrictedTestCase::validateSettings(_evmVersion);
-}
-
 TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
 {
 	for(bool compileViaYul: set<bool>{!m_runWithoutYul, m_runWithYul})

test/libsolidity/SemanticTest.h

@@ -44,8 +44,6 @@ public:
 
 	explicit SemanticTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
 
-	bool validateSettings(langutil::EVMVersion _evmVersion) override;
-
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
 	void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;

test/libyul/SyntaxTest.cpp

@@ -114,13 +114,12 @@ void SyntaxTest::parseAndAnalyze()
 
 }
 
-bool SyntaxTest::validateSettings(langutil::EVMVersion _evmVersion)
+void SyntaxTest::validateSettings()
 {
-	if (!CommonSyntaxTest::validateSettings(_evmVersion))
+	CommonSyntaxTest::validateSettings();
-		return false;
 
 	if (!m_settings.count("dialect"))
-		return true;
+		return;
 
 	string const dialect = m_settings["dialect"];
 	m_validatedSettings["dialect"] = dialect;
@@ -134,6 +133,4 @@ bool SyntaxTest::validateSettings(langutil::EVMVersion _evmVersion)
 			joinHumanReadable(validDialectNames(), ", ", " and ") +
 			"."
 		});
-
-	return true;
 }

test/libyul/SyntaxTest.h

@@ -42,10 +42,7 @@ public:
 
 	/// Validates the settings, i.e. moves them from m_settings to m_validatedSettings.
 	/// Throws a runtime exception if any setting is left at this class (i.e. unknown setting).
-	/// Returns true, if the test case is supported in the current environment and false
+	void validateSettings() override;
-	/// otherwise which causes this test to be skipped.
-	/// This might check e.g. for restrictions on the EVM version.
-	bool validateSettings(langutil::EVMVersion _evmVersion) override;
 protected:
 	void parseAndAnalyze() override;
 };

test/tools/isoltest.cpp

@@ -161,7 +161,8 @@ TestTool::Result TestTool::process()
 			(AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush();
 
 			m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.evmVersion()});
-			if (m_test->validateSettings(m_options.evmVersion()))
+			m_test->validateSettings();
+			if (m_test->shouldRun())
 				switch (TestCase::TestResult result = m_test->run(outputMessages, "  ", formatted))
 				{
 					case TestCase::TestResult::Success: