Add settings framework for interactive tests.

test/TestCase.cpp

@@ -17,76 +17,80 @@
 
 #include <test/TestCase.h>
 
+#include <libdevcore/StringUtils.h>
+
 #include <boost/algorithm/cxx11/none_of.hpp>
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/trim.hpp>
+#include <boost/range/adaptor/map.hpp>
 
 #include <stdexcept>
 
+#include <iostream>
+
 using namespace dev;
 using namespace solidity;
 using namespace dev::solidity::test;
 using namespace std;
 
+void TestCase::printUpdatedSettings(ostream& _stream, const string& _linePrefix, const bool) const
+{
+	if (m_validatedSettings.empty())
+		return;
+
+	_stream << _linePrefix << "// ====" << endl;
+	for (auto const& setting: m_validatedSettings)
+		_stream << _linePrefix << "// " << setting.first << ": " << setting.second << endl;
+}
+
 bool TestCase::isTestFilename(boost::filesystem::path const& _filename)
 {
 	string extension = _filename.extension().string();
 	return (extension == ".sol" || extension == ".yul") &&
 		   !boost::starts_with(_filename.string(), "~") &&
-		   !boost::starts_with(_filename.string(), ".");
+			!boost::starts_with(_filename.string(), ".");
 }
 
-bool TestCase::supportedForEVMVersion(langutil::EVMVersion _evmVersion) const
+bool TestCase::validateSettings(langutil::EVMVersion)
 {
-	return boost::algorithm::none_of(m_evmVersionRules, [&](auto const& rule) { return !rule(_evmVersion); });
+	if (!m_settings.empty())
+		throw runtime_error(
+			"Unknown setting(s): " +
+			joinHumanReadable(m_settings | boost::adaptors::map_keys)
+		);
+	return true;
 }
 
-string TestCase::parseSource(istream& _stream)
+string TestCase::parseSourceAndSettings(istream& _stream)
 {
 	string source;
 	string line;
+	static string const settingsDelimiter("// ====");
 	static string const delimiter("// ----");
-	static string const evmVersion("// EVMVersion: ");
+	bool sourcePart = true;
-	bool isTop = true;
 	while (getline(_stream, line))
+	{
 		if (boost::algorithm::starts_with(line, delimiter))
 			break;
-		else
+		else if (boost::algorithm::starts_with(line, settingsDelimiter))
-		{
+			sourcePart = false;
-			if (isTop && boost::algorithm::starts_with(line, evmVersion))
+		else if (sourcePart)
-			{
-				string versionString = line.substr(evmVersion.size() + 1);
-				auto version = langutil::EVMVersion::fromString(versionString);
-				if (!version)
-					throw runtime_error("Invalid EVM version: \"" + versionString + "\"");
-				switch (line.at(evmVersion.size()))
-				{
-					case '>':
-						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
-							return version < _version;
-						});
-						break;
-					case '<':
-						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
-							return _version < version;
-						});
-						break;
-					case '=':
-						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
-							return _version == version;
-						});
-						break;
-					case '!':
-						m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
-							return !(_version == version);
-						});
-						break;
-				}
-			}
-			else
-				isTop = false;
 			source += line + "\n";
+		else if (boost::algorithm::starts_with(line, "// "))
+		{
+			size_t colon = line.find(':');
+			if (colon == string::npos)
+				throw runtime_error(string("Expected \":\" inside setting."));
+			string key = line.substr(3, colon - 3);
+			string value = line.substr(colon + 1);
+			boost::algorithm::trim(key);
+			boost::algorithm::trim(value);
+			m_settings[key] = value;
 		}
+		else
+			throw runtime_error(string("Expected \"//\" or \"// ---\" to terminate settings and source."));
+	}
 	return source;
 }
 
@@ -96,3 +100,35 @@ void TestCase::expect(string::iterator& _it, string::iterator _end, string::valu
 		throw runtime_error(string("Invalid test expectation. Expected: \"") + _c + "\".");
 	++_it;
 }
+
+bool EVMVersionRestrictedTestCase::validateSettings(langutil::EVMVersion _evmVersion)
+{
+	if (!m_settings.count("EVMVersion"))
+		return true;
+
+	string versionString = m_settings["EVMVersion"];
+	m_validatedSettings["EVMVersion"] = versionString;
+	m_settings.erase("EVMVersion");
+
+	if (!TestCase::validateSettings(_evmVersion))
+		return false;
+
+	if (versionString.empty())
+		return true;
+
+	char comparator = versionString.front();
+	versionString = versionString.substr(1);
+	boost::optional<langutil::EVMVersion> version = langutil::EVMVersion::fromString(versionString);
+	if (!version)
+		throw runtime_error("Invalid EVM version: \"" + versionString + "\"");
+
+	switch (comparator)
+	{
+		case '>': return version < _evmVersion;
+		case '<': return _evmVersion < version;
+		case '=': return _evmVersion == version;
+		case '!': return !(_evmVersion == version);
+	}
+	throw runtime_error(string{"Invalid EVM comparator: \""} + comparator + "\"");
+	return false; // not reached
+}

test/TestCase.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <map>
 
 namespace dev
 {
@@ -42,7 +43,9 @@ namespace test
 	} \
 	while (false)
 
-/** Common superclass of SyntaxTest and SemanticsTest. */
+/**
+ * Common superclass of anything that can be run via isoltest.
+ */
 class TestCase
 {
 public:
@@ -68,17 +71,23 @@ public:
 	/// If @arg _formatted is true, color-coding may be used to indicate
 	/// error locations in the contract, if applicable.
 	virtual void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const = 0;
+	/// Outputs the updated settings.
+	virtual void printUpdatedSettings(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const;
 	/// Outputs test expectations to @arg _stream that match the actual results of the test.
 	/// Each line of output is prefixed with @arg _linePrefix.
 	virtual void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const = 0;
 
 	static bool isTestFilename(boost::filesystem::path const& _filename);
 
-	/// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise.
+	/// Validates the settings, i.e. moves them from m_settings to m_validatedSettings.
-	bool supportedForEVMVersion(langutil::EVMVersion _evmVersion) const;
+	/// Throws a runtime exception if any setting is left at this class (i.e. unknown setting).
+	/// Returns true, if the test case is supported in the current environment and false
+	/// otherwise which causes this test to be skipped.
+	/// This might check e.g. for restrictions on the EVM version.
+	virtual bool validateSettings(langutil::EVMVersion /*_evmVersion*/);
 
 protected:
-	std::string parseSource(std::istream& _file);
+	std::string parseSourceAndSettings(std::istream& _file);
 	static void expect(std::string::iterator& _it, std::string::iterator _end, std::string::value_type _c);
 
 	template<typename IteratorType>
@@ -94,10 +103,19 @@ protected:
 		while (_it != _end && *_it == '/')
 			++_it;
 	}
-private:
+
-	std::vector<std::function<bool(langutil::EVMVersion)>> m_evmVersionRules;
+	/// Parsed settings.
+	std::map<std::string, std::string> m_settings;
+	/// Updated settings after validation.
+	std::map<std::string, std::string> m_validatedSettings;
 };
 
+class EVMVersionRestrictedTestCase: public TestCase
+{
+public:
+	/// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise.
+	bool validateSettings(langutil::EVMVersion _evmVersion) override;
+};
 }
 }
 }

test/boostTest.cpp

@@ -105,7 +105,7 @@ int registerTests(
 					{
 						stringstream errorStream;
 						auto testCase = _testCaseCreator(config);
-						if (testCase->supportedForEVMVersion(dev::test::Options::get().evmVersion()))
+						if (testCase->validateSettings(dev::test::Options::get().evmVersion()))
 							if (!testCase->run(errorStream))
 								BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
 					}

test/libsolidity/SemanticTest.cpp

@@ -43,7 +43,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, lang
 	soltestAssert(file, "Cannot open test contract: \"" + _filename + "\".");
 	file.exceptions(ios::badbit);
 
-	m_source = parseSource(file);
+	m_source = parseSourceAndSettings(file);
 	parseExpectations(file);
 }
 

test/libsolidity/SemanticTest.h

@@ -40,7 +40,7 @@ namespace test
  * section from the given file. This comment section should define a set of functions to be called
  * and an expected result they return after being executed.
  */
-class SemanticTest: public SolidityExecutionFramework, public TestCase
+class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrictedTestCase
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _options)

test/libsolidity/SyntaxTest.cpp

@@ -59,7 +59,7 @@ SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion
 		BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\"."));
 	file.exceptions(ios::badbit);
 
-	m_source = parseSource(file);
+	m_source = parseSourceAndSettings(file);
 	m_expectations = parseExpectations(file);
 }
 

test/libsolidity/SyntaxTest.h

@@ -50,7 +50,7 @@ struct SyntaxTestError
 };
 
 
-class SyntaxTest: AnalysisFramework, public TestCase
+class SyntaxTest: AnalysisFramework, public EVMVersionRestrictedTestCase
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _config)

test/libsolidity/semanticTests/shifts.sol

@@ -0,0 +1,9 @@
+contract C {
+    function f(uint x) public returns (uint y) {
+        assembly { y := shl(2, x) }
+    }
+}
+// ====
+// EVMVersion: >byzantium
+// ----
+// f(uint256): 7 -> 28

test/libyul/YulOptimizerTest.h

@@ -38,7 +38,7 @@ namespace yul
 namespace test
 {
 
-class YulOptimizerTest: public dev::solidity::test::TestCase
+class YulOptimizerTest: public dev::solidity::test::EVMVersionRestrictedTestCase
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _config)

test/tools/isoltest.cpp

@@ -126,7 +126,7 @@ TestTool::Result TestTool::process()
 	try
 	{
 		m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
-		if (m_test->supportedForEVMVersion(m_evmVersion))
+		if (m_test->validateSettings(m_evmVersion))
 			success = m_test->run(outputMessages, "  ", m_formatted);
 		else
 		{
@@ -164,6 +164,7 @@ TestTool::Result TestTool::process()
 
 		AnsiColorized(cout, m_formatted, {BOLD, CYAN}) << "  Contract:" << endl;
 		m_test->printSource(cout, "    ", m_formatted);
+		m_test->printUpdatedSettings(cout, "    ", m_formatted);
 
 		cout << endl << outputMessages.str() << endl;
 		return Result::Failure;
@@ -193,6 +194,7 @@ TestTool::Request TestTool::handleResponse(bool _exception)
 				cout << endl;
 				ofstream file(m_path.string(), ios::trunc);
 				m_test->printSource(file);
+				m_test->printUpdatedSettings(file);
 				file << "// ----" << endl;
 				m_test->printUpdatedExpectations(file, "// ");
 				return Request::Rerun;