Support EVM Version rules for extracted tests.

test/Common.cpp

@@ -75,6 +75,7 @@ CommonOptions::CommonOptions(std::string _caption):
 	)
 {
 	options.add_options()
+		("evm-version", po::value(&evmVersionString), "which evm version to use")
 		("testpath", po::value<fs::path>(&this->testPath)->default_value(dev::test::testPath()), "path to test files")
 		("ipcpath", po::value<fs::path>(&ipcPath)->default_value(IPCEnvOrDefaultPath()), "path to ipc socket")
 		("no-ipc", po::bool_switch(&disableIPC), "disable semantic tests")
@@ -121,6 +122,20 @@ bool CommonOptions::parse(int argc, char const* const* argv)
 	return true;
 }
 
+
+langutil::EVMVersion CommonOptions::evmVersion() const
+{
+	if (!evmVersionString.empty())
+	{
+		auto version = langutil::EVMVersion::fromString(evmVersionString);
+		if (!version)
+			throw std::runtime_error("Invalid EVM version: " + evmVersionString);
+		return *version;
+	}
+	else
+		return langutil::EVMVersion();
+}
+
 }
 
 }

test/Common.h

@@ -18,6 +18,7 @@
 #pragma once
 
 #include <libdevcore/Exceptions.h>
+#include <liblangutil/EVMVersion.h>
 
 #include <boost/filesystem/path.hpp>
 #include <boost/program_options.hpp>
@@ -39,6 +40,8 @@ struct CommonOptions: boost::noncopyable
 	bool disableIPC = false;
 	bool disableSMT = false;
 
+	langutil::EVMVersion evmVersion() const;
+
 	virtual bool parse(int argc, char const* const* argv);
 	// Throws a ConfigException on error
 	virtual void validate() const;
@@ -47,6 +50,9 @@ protected:
 	CommonOptions(std::string caption = "");
 
 	boost::program_options::options_description options;
+
+private:
+	std::string evmVersionString;
 };
 
 }

test/ExecutionFramework.cpp

@@ -50,13 +50,13 @@ string getIPCSocketPath()
 }
 
 ExecutionFramework::ExecutionFramework():
-	ExecutionFramework(getIPCSocketPath())
+	ExecutionFramework(getIPCSocketPath(), dev::test::Options::get().evmVersion())
 {
 }
 
-ExecutionFramework::ExecutionFramework(string const& _ipcPath):
+ExecutionFramework::ExecutionFramework(string const& _ipcPath, langutil::EVMVersion const _evmVersion):
 	m_rpc(RPCSession::instance(_ipcPath)),
-	m_evmVersion(dev::test::Options::get().evmVersion()),
+	m_evmVersion(_evmVersion),
 	m_optimize(dev::test::Options::get().optimize),
 	m_showMessages(dev::test::Options::get().showMessages),
 	m_sender(m_rpc.account(0))

test/ExecutionFramework.h

@@ -53,7 +53,7 @@ class ExecutionFramework
 
 public:
 	ExecutionFramework();
-	explicit ExecutionFramework(std::string const& _ipcPath);
+	explicit ExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion const _evmVersion);
 	virtual ~ExecutionFramework() = default;
 
 	virtual bytes const& compileAndRunWithoutCheck(

test/Options.cpp

@@ -53,22 +53,7 @@ Options::Options()
 	options.add_options()
 		("optimize", po::bool_switch(&optimize), "enables optimization")
 		("abiencoderv2", po::bool_switch(&useABIEncoderV2), "enables abi encoder v2")
-		("evm-version", po::value(&evmVersionString), "which evm version to use")
 		("show-messages", po::bool_switch(&showMessages), "enables message output");
 
 	parse(suite.argc, suite.argv);
 }
-
-langutil::EVMVersion Options::evmVersion() const
-{
-	if (!evmVersionString.empty())
-	{
-		// We do this check as opposed to in the constructor because the BOOST_REQUIRE
-		// macros cannot yet be used in the constructor.
-		auto version = langutil::EVMVersion::fromString(evmVersionString);
-		BOOST_REQUIRE_MESSAGE(version, "Invalid EVM version: " + evmVersionString);
-		return *version;
-	}
-	else
-		return langutil::EVMVersion();
-}

test/Options.h

@@ -37,13 +37,9 @@ struct Options: CommonOptions
 	bool showMessages = false;
 	bool useABIEncoderV2 = false;
 
-	langutil::EVMVersion evmVersion() const;
-
 	static Options const& get();
 
 private:
-	std::string evmVersionString;
-
 	Options();
 };
 

test/TestCase.cpp

@@ -35,16 +35,60 @@ bool TestCase::isTestFilename(boost::filesystem::path const& _filename)
 		   !boost::starts_with(_filename.string(), ".");
 }
 
+bool TestCase::supportedForEVMVersion(langutil::EVMVersion _evmVersion) const
+{
+	for (auto const& rule: m_evmVersionRules)
+		if (!rule(_evmVersion))
+			return false;
+	return true;
+}
+
 string TestCase::parseSource(istream& _stream)
 {
 	string source;
 	string line;
 	string const delimiter("// ----");
+	string const evmVersion("// EVMVersion: ");
+	bool isTop = true;
 	while (getline(_stream, line))
 		if (boost::algorithm::starts_with(line, delimiter))
 			break;
 		else
+		{
+			if (isTop && boost::algorithm::starts_with(line, evmVersion))
+			{
+				string versionString = line.substr(evmVersion.size() + 1);
+				auto version = langutil::EVMVersion::fromString(versionString);
+				if (!version)
+					throw runtime_error("Invalid EVM version: \"" + versionString + "\"");
+				switch (line.at(evmVersion.size()))
+				{
+					case '>':
+						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
+							return version < _version;
+						});
+						break;
+					case '<':
+						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
+							return _version < version;
+						});
+						break;
+					case '=':
+						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
+							return _version == version;
+						});
+						break;
+					case '!':
+						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
+							return !(_version == version);
+						});
+						break;
+				}
+			}
+			else
+				isTop = false;
 			source += line + "\n";
+		}
 	return source;
 }
 

test/TestCase.h

@@ -17,11 +17,15 @@
 
 #pragma once
 
+#include <liblangutil/EVMVersion.h>
+
 #include <boost/filesystem.hpp>
 
+#include <functional>
 #include <iosfwd>
 #include <memory>
 #include <string>
+#include <vector>
 
 namespace dev
 {
@@ -46,6 +50,7 @@ public:
 	{
 		std::string filename;
 		std::string ipcPath;
+		langutil::EVMVersion evmVersion;
 	};
 
 	using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
@@ -69,8 +74,11 @@ public:
 
 	static bool isTestFilename(boost::filesystem::path const& _filename);
 
+	/// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise.
+	bool supportedForEVMVersion(langutil::EVMVersion _evmVersion) const;
+
 protected:
-	static std::string parseSource(std::istream& _file);
+	std::string parseSource(std::istream& _file);
 	static void expect(std::string::iterator& _it, std::string::iterator _end, std::string::value_type _c);
 
 	template<typename IteratorType>
@@ -86,7 +94,8 @@ protected:
 		while (_it != _end && *_it == '/')
 			++_it;
 	}
-
+private:
+	std::vector<std::function<bool(langutil::EVMVersion)>> m_evmVersionRules;
 };
 
 }

test/boostTest.cpp

@@ -80,7 +80,7 @@ int registerTests(
 {
 	int numTestsAdded = 0;
 	fs::path fullpath = _basepath / _path;
-	TestCase::Config config{fullpath.string(), _ipcPath};
+	TestCase::Config config{fullpath.string(), _ipcPath, dev::test::Options::get().evmVersion()};
 	if (fs::is_directory(fullpath))
 	{
 		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
@@ -104,7 +104,9 @@ int registerTests(
 					try
 					{
 						stringstream errorStream;
-						if (!_testCaseCreator(config)->run(errorStream))
+						auto testCase = _testCaseCreator(config);
+						if (testCase->supportedForEVMVersion(dev::test::Options::get().evmVersion()))
+							if (!testCase->run(errorStream))
 								BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
 					}
 					catch (boost::exception const& _e)

test/libsolidity/SMTCheckerJSONTest.cpp

@@ -35,8 +35,8 @@ using namespace dev;
 using namespace std;
 using namespace boost::unit_test;
 
-SMTCheckerTest::SMTCheckerTest(string const& _filename)
+SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion const _evmVersion)
-: SyntaxTest(_filename)
+: SyntaxTest(_filename, _evmVersion)
 {
 	if (!boost::algorithm::ends_with(_filename, ".sol"))
 		BOOST_THROW_EXCEPTION(runtime_error("Invalid test contract file name: \"" + _filename + "\"."));

test/libsolidity/SMTCheckerJSONTest.h

@@ -35,9 +35,9 @@ class SMTCheckerTest: public SyntaxTest
 public:
 	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new SMTCheckerTest(_config.filename));
+		return std::unique_ptr<TestCase>(new SMTCheckerTest(_config.filename, _config.evmVersion));
 	}
-	SMTCheckerTest(std::string const& _filename);
+	SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 

test/libsolidity/SemanticTest.cpp

@@ -36,8 +36,8 @@ using namespace boost::unit_test;
 namespace fs = boost::filesystem;
 
 
-SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
+SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, langutil::EVMVersion const _evmVersion):
-	SolidityExecutionFramework(_ipcPath)
+	SolidityExecutionFramework(_ipcPath, _evmVersion)
 {
 	ifstream file(_filename);
 	soltestAssert(file, "Cannot open test contract: \"" + _filename + "\".");

test/libsolidity/SemanticTest.h

@@ -44,9 +44,9 @@ class SemanticTest: public SolidityExecutionFramework, public TestCase
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _options)
-	{ return std::make_unique<SemanticTest>(_options.filename, _options.ipcPath); }
+	{ return std::make_unique<SemanticTest>(_options.filename, _options.ipcPath, _options.evmVersion); }
 
-	explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath);
+	explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion const _evmVersion);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;

test/libsolidity/SolidityExecutionFramework.cpp

@@ -33,7 +33,7 @@ SolidityExecutionFramework::SolidityExecutionFramework():
 {
 }
 
-SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath):
+SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion const _evmVersion):
-	ExecutionFramework(_ipcPath)
+	ExecutionFramework(_ipcPath, _evmVersion)
 {
 }

test/libsolidity/SolidityExecutionFramework.h

@@ -43,7 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework
 
 public:
 	SolidityExecutionFramework();
-	SolidityExecutionFramework(std::string const& _ipcPath);
+	SolidityExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion const _evmVersion);
 
 	virtual bytes const& compileAndRunWithoutCheck(
 		std::string const& _sourceCode,

test/libsolidity/SyntaxTest.cpp

@@ -52,7 +52,7 @@ int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
 
 }
 
-SyntaxTest::SyntaxTest(string const& _filename)
+SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion const _evmVersion): m_evmVersion(_evmVersion)
 {
 	ifstream file(_filename);
 	if (!file)
@@ -68,7 +68,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _fo
 	string const versionPragma = "pragma solidity >=0.0;\n";
 	m_compiler.reset();
 	m_compiler.addSource("", versionPragma + m_source);
-	m_compiler.setEVMVersion(dev::test::Options::get().evmVersion());
+	m_compiler.setEVMVersion(m_evmVersion);
 
 	if (m_compiler.parse())
 		m_compiler.analyze();

test/libsolidity/SyntaxTest.h

@@ -54,8 +54,8 @@ class SyntaxTest: AnalysisFramework, public TestCase
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _config)
-	{ return std::unique_ptr<TestCase>(new SyntaxTest(_config.filename)); }
+	{ return std::unique_ptr<TestCase>(new SyntaxTest(_config.filename, _config.evmVersion)); }
-	SyntaxTest(std::string const& _filename);
+	SyntaxTest(std::string const& _filename, langutil::EVMVersion const _evmVersion);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
 
@@ -82,6 +82,7 @@ protected:
 	std::string m_source;
 	std::vector<SyntaxTestError> m_expectations;
 	std::vector<SyntaxTestError> m_errorList;
+	langutil::EVMVersion const m_evmVersion;
 };
 
 }

test/tools/IsolTestOptions.h

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <liblangutil/EVMVersion.h>
+
 #include <test/Common.h>
 
 namespace dev

test/tools/isoltest.cpp

@@ -47,13 +47,15 @@ namespace fs = boost::filesystem;
 
 struct TestStats
 {
-	int successCount;
+	int successCount = 0;
-	int testCount;
+	int testCount = 0;
-	operator bool() const { return successCount == testCount; }
+	int skippedCount = 0;
+	operator bool() const { return successCount + skippedCount == testCount; }
 	TestStats& operator+=(TestStats const& _other) noexcept
 	{
 		successCount += _other.successCount;
 		testCount += _other.testCount;
+		skippedCount += _other.skippedCount;
 		return *this;
 	}
 };
@@ -66,15 +68,17 @@ public:
 		string const& _name,
 		fs::path const& _path,
 		string const& _ipcPath,
-		bool _formatted
+		bool _formatted,
-	): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted)
+		langutil::EVMVersion _evmVersion
+	): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion)
 	{}
 
 	enum class Result
 	{
 		Success,
 		Failure,
-		Exception
+		Exception,
+		Skipped
 	};
 
 	Result process();
@@ -84,7 +88,8 @@ public:
 		fs::path const& _basepath,
 		fs::path const& _path,
 		string const& _ipcPath,
-		bool const _formatted
+		bool const _formatted,
+		langutil::EVMVersion const _evmVersion
 	);
 
 	static string editor;
@@ -103,6 +108,7 @@ private:
 	fs::path const m_path;
 	string m_ipcPath;
 	bool const m_formatted = false;
+	langutil::EVMVersion const m_evmVersion;
 	unique_ptr<TestCase> m_test;
 	static bool m_exitRequested;
 };
@@ -119,8 +125,14 @@ TestTool::Result TestTool::process()
 
 	try
 	{
-		m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath});
+		m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
+		if (m_test->supportedForEVMVersion(m_evmVersion))
 			success = m_test->run(outputMessages, "  ", m_formatted);
+		else
+		{
+			AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
+			return Result::Skipped;
+		}
 	}
 	catch(boost::exception const& _e)
 	{
@@ -204,13 +216,15 @@ TestStats TestTool::processPath(
 	fs::path const& _basepath,
 	fs::path const& _path,
 	string const& _ipcPath,
-	bool const _formatted
+	bool const _formatted,
+	langutil::EVMVersion const _evmVersion
 )
 {
 	std::queue<fs::path> paths;
 	paths.push(_path);
 	int successCount = 0;
 	int testCount = 0;
+	int skippedCount = 0;
 
 	while (!paths.empty())
 	{
@@ -235,7 +249,7 @@ TestStats TestTool::processPath(
 		else
 		{
 			++testCount;
-			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted);
+			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion);
 			auto result = testTool.process();
 
 			switch(result)
@@ -254,6 +268,7 @@ TestStats TestTool::processPath(
 					break;
 				case Request::Skip:
 					paths.pop();
+					++skippedCount;
 					break;
 				}
 				break;
@@ -261,11 +276,15 @@ TestStats TestTool::processPath(
 				paths.pop();
 				++successCount;
 				break;
+			case Result::Skipped:
+				paths.pop();
+				++skippedCount;
+				break;
 			}
 		}
 	}
 
-	return { successCount, testCount };
+	return { successCount, testCount, skippedCount };
 
 }
 
@@ -298,7 +317,8 @@ boost::optional<TestStats> runTestSuite(
 	fs::path const& _subdirectory,
 	string const& _ipcPath,
 	TestCase::TestCaseCreator _testCaseCreator,
-	bool _formatted
+	bool _formatted,
+	langutil::EVMVersion const _evmVersion
 )
 {
 	fs::path testPath = _basePath / _subdirectory;
@@ -309,14 +329,21 @@ boost::optional<TestStats> runTestSuite(
 		return {};
 	}
 
-	TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted);
+	TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion);
 
 	cout << endl << _name << " Test Summary: ";
 	AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
 		stats.successCount <<
 		"/" <<
 		stats.testCount;
-	cout << " tests successful." << endl << endl;
+	cout << " tests successful";
+	if (stats.skippedCount > 0)
+	{
+		cout << " (";
+		AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount;
+		cout<< " tests skipped)";
+	}
+	cout << "." << endl << endl;
 
 	return stats;
 }
@@ -354,7 +381,7 @@ int main(int argc, char const *argv[])
 		if (ts.smt && options.disableSMT)
 			continue;
 
-		if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor))
+		if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion()))
 			global_stats += *stats;
 		else
 			return 1;
@@ -363,7 +390,14 @@ int main(int argc, char const *argv[])
 	cout << endl << "Summary: ";
 	AnsiColorized(cout, !options.noColor, {BOLD, global_stats ? GREEN : RED}) <<
 		 global_stats.successCount << "/" << global_stats.testCount;
-	cout << " tests successful." << endl;
+	cout << " tests successful";
+	if (global_stats.skippedCount > 0)
+	{
+		cout << " (";
+		AnsiColorized(cout, !options.noColor, {BOLD, YELLOW}) << global_stats.skippedCount;
+		cout<< " tests skipped)";
+	}
+	cout << "." << endl;
 
 	return global_stats ? 0 : 1;
 }