Merge pull request #6500 from ethereum/soltest-case-selector

[soltest] Test filter for isoltest

Changelog.md

@@ -27,6 +27,8 @@ Bugfixes:
  * Yul: Properly register functions and disallow shadowing between function variables and variables in the outside scope.
 
 
+Build System:
+ * Soltest: Add commandline option `--test` / `-t` to isoltest which takes a string that allows filtering unit tests.
 
 
 ### 0.5.7 (2019-03-26)

test/tools/IsolTestOptions.cpp

@@ -19,9 +19,14 @@
 */
 
 #include <test/tools/IsolTestOptions.h>
+
+#include <libdevcore/Assertions.h>
+
 #include <boost/filesystem.hpp>
-#include <string>
+
 #include <iostream>
+#include <regex>
+#include <string>
 
 namespace fs = boost::filesystem;
 namespace po = boost::program_options;
@@ -32,7 +37,7 @@ namespace test
 {
 
 auto const description = R"(isoltest, tool for interactively managing test contracts.
-Usage: isoltest [Options] --ipcpath ipcpath
+Usage: isoltest [Options]
 Interactively validates test contracts.
 
 Allowed options)";
@@ -51,10 +56,10 @@ IsolTestOptions::IsolTestOptions(std::string* _editor):
 	CommonOptions(description)
 {
 	options.add_options()
+		("editor", po::value<std::string>(_editor)->default_value(editorPath()), "Path to editor for opening test files.")
 		("help", po::bool_switch(&showHelp), "Show this help screen.")
-		("no-color", po::bool_switch(&noColor), "don't use colors")
+		("no-color", po::bool_switch(&noColor), "Don't use colors.")
-		("editor", po::value<std::string>(_editor)->default_value(editorPath()), "editor for opening test files");
+		("test,t", po::value<std::string>(&testFilter)->default_value("*/*"), "Filters which test units to include.");
-
 }
 
 bool IsolTestOptions::parse(int _argc, char const* const* _argv)
@@ -70,5 +75,16 @@ bool IsolTestOptions::parse(int _argc, char const* const* _argv)
 	return res;
 }
 
+void IsolTestOptions::validate() const
+{
+	static std::string filterString{"[a-zA-Z1-9_/*]*"};
+	static std::regex filterExpression{filterString};
+	assertThrow(
+		regex_match(testFilter, filterExpression),
+		ConfigException,
+		"Invalid test unit filter - can only contain '" + filterString + ": " + testFilter
+	);
+}
+
 }
 }

test/tools/IsolTestOptions.h

@@ -30,11 +30,13 @@ namespace test
 
 struct IsolTestOptions: CommonOptions
 {
-	bool noColor = false;
 	bool showHelp = false;
+	bool noColor = false;
+	std::string testFilter = std::string{};
 
 	IsolTestOptions(std::string* _editor);
 	bool parse(int _argc, char const* const* _argv) override;
+	void validate() const override;
 };
 }
 }

test/tools/isoltest.cpp

@@ -32,6 +32,7 @@
 #include <iostream>
 #include <fstream>
 #include <queue>
+#include <regex>
 
 #if defined(_WIN32)
 #include <windows.h>
@@ -45,6 +46,9 @@ using namespace std;
 namespace po = boost::program_options;
 namespace fs = boost::filesystem;
 
+using TestCreator = TestCase::TestCaseCreator;
+using TestOptions = dev::test::IsolTestOptions;
+
 struct TestStats
 {
 	int successCount = 0;
@@ -60,17 +64,43 @@ struct TestStats
 	}
 };
 
+class TestFilter
+{
+public:
+	explicit TestFilter(string const& _filter): m_filter(_filter)
+	{
+		string filter{m_filter};
+
+		boost::replace_all(filter, "/", "\\/");
+		boost::replace_all(filter, "*", ".*");
+
+		m_filterExpression = regex{"(" + filter + "(\\.sol|\\.yul))"};
+	}
+
+	bool matches(string const& _name) const
+	{
+		return regex_match(_name, m_filterExpression);
+	}
+
+private:
+	string m_filter;
+	regex m_filterExpression;
+};
+
 class TestTool
 {
 public:
 	TestTool(
-		TestCase::TestCaseCreator _testCaseCreator,
+		TestCreator _testCaseCreator,
-		string const& _name,
+		TestOptions const& _options,
 		fs::path const& _path,
-		string const& _ipcPath,
+		string const& _name
-		bool _formatted,
+	):
-		langutil::EVMVersion _evmVersion
+		m_testCaseCreator(_testCaseCreator),
-	): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion)
+		m_options(_options),
+		m_filter(TestFilter{_options.testFilter}),
+		m_path(_path),
+		m_name(_name)
 	{}
 
 	enum class Result
@@ -84,12 +114,10 @@ public:
 	Result process();
 
 	static TestStats processPath(
-		TestCase::TestCaseCreator _testCaseCreator,
+		TestCreator _testCaseCreator,
+		TestOptions const& _options,
 		fs::path const& _basepath,
-		fs::path const& _path,
+		fs::path const& _path
-		string const& _ipcPath,
-		bool _formatted,
-		langutil::EVMVersion _evmVersion
 	);
 
 	static string editor;
@@ -103,13 +131,14 @@ private:
 
 	Request handleResponse(bool _exception);
 
-	TestCase::TestCaseCreator m_testCaseCreator;
+	TestCreator m_testCaseCreator;
-	string const m_name;
+	TestOptions const& m_options;
+	TestFilter m_filter;
 	fs::path const m_path;
-	string m_ipcPath;
+	string const m_name;
-	bool const m_formatted = false;
+
-	langutil::EVMVersion const m_evmVersion;
 	unique_ptr<TestCase> m_test;
+
 	static bool m_exitRequested;
 };
 
@@ -119,52 +148,58 @@ bool TestTool::m_exitRequested = false;
 TestTool::Result TestTool::process()
 {
 	bool success;
+	bool formatted{!m_options.noColor};
 	std::stringstream outputMessages;
 
-	(AnsiColorized(cout, m_formatted, {BOLD}) << m_name << ": ").flush();
-
 	try
 	{
-		m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
+		if (m_filter.matches(m_name))
-		if (m_test->validateSettings(m_evmVersion))
-			success = m_test->run(outputMessages, "  ", m_formatted);
-		else
 		{
-			AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
+			(AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush();
-			return Result::Skipped;
+
+			m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
+			if (m_test->validateSettings(m_options.evmVersion()))
+				success = m_test->run(outputMessages, "  ", formatted);
+			else
+			{
+				AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
+				return Result::Skipped;
+			}
 		}
+		else
+			return Result::Skipped;
 	}
 	catch(boost::exception const& _e)
 	{
-		AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
+		AnsiColorized(cout, formatted, {BOLD, RED}) <<
 			"Exception during test: " << boost::diagnostic_information(_e) << endl;
 		return Result::Exception;
 	}
 	catch (std::exception const& _e)
 	{
-		AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
+		AnsiColorized(cout, formatted, {BOLD, RED}) <<
 			"Exception during test: " << _e.what() << endl;
 		return Result::Exception;
 	}
 	catch (...)
 	{
-		AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
+		AnsiColorized(cout, formatted, {BOLD, RED}) <<
 			"Unknown exception during test." << endl;
 		return Result::Exception;
 	}
 
 	if (success)
 	{
-		AnsiColorized(cout, m_formatted, {BOLD, GREEN}) << "OK" << endl;
+		AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
 		return Result::Success;
 	}
 	else
 	{
-		AnsiColorized(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl;
+		AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
 
-		AnsiColorized(cout, m_formatted, {BOLD, CYAN}) << "  Contract:" << endl;
+		AnsiColorized(cout, formatted, {BOLD, CYAN}) << "  Contract:" << endl;
-		m_test->printSource(cout, "    ", m_formatted);
+		m_test->printSource(cout, "    ", formatted);
-		m_test->printUpdatedSettings(cout, "    ", m_formatted);
+		m_test->printUpdatedSettings(cout, "    ", formatted);
 
 		cout << endl << outputMessages.str() << endl;
 		return Result::Failure;
@@ -214,12 +249,10 @@ TestTool::Request TestTool::handleResponse(bool _exception)
 }
 
 TestStats TestTool::processPath(
-	TestCase::TestCaseCreator _testCaseCreator,
+	TestCreator _testCaseCreator,
+	TestOptions const& _options,
 	fs::path const& _basepath,
-	fs::path const& _path,
+	fs::path const& _path
-	string const& _ipcPath,
-	bool _formatted,
-	langutil::EVMVersion _evmVersion
 )
 {
 	std::queue<fs::path> paths;
@@ -251,7 +284,12 @@ TestStats TestTool::processPath(
 		else
 		{
 			++testCount;
-			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion);
+			TestTool testTool(
+				_testCaseCreator,
+				_options,
+				fullpath,
+				currentPath.string()
+			);
 			auto result = testTool.process();
 
 			switch(result)
@@ -314,16 +352,15 @@ void setupTerminal()
 }
 
 boost::optional<TestStats> runTestSuite(
-	string const& _name,
+	TestCreator _testCaseCreator,
+	TestOptions const& _options,
 	fs::path const& _basePath,
 	fs::path const& _subdirectory,
-	string const& _ipcPath,
+	string const& _name
-	TestCase::TestCaseCreator _testCaseCreator,
-	bool _formatted,
-	langutil::EVMVersion _evmVersion
 )
 {
-	fs::path testPath = _basePath / _subdirectory;
+	fs::path testPath{_basePath / _subdirectory};
+	bool formatted{!_options.noColor};
 
 	if (!fs::exists(testPath) || !fs::is_directory(testPath))
 	{
@@ -331,22 +368,29 @@ boost::optional<TestStats> runTestSuite(
 		return {};
 	}
 
-	TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion);
+	TestStats stats = TestTool::processPath(
+		_testCaseCreator,
+		_options,
+		_basePath,
+		_subdirectory
+	);
 
-	cout << endl << _name << " Test Summary: ";
+	if (stats.skippedCount != stats.testCount)
-	AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
-		stats.successCount <<
-		"/" <<
-		stats.testCount;
-	cout << " tests successful";
-	if (stats.skippedCount > 0)
 	{
-		cout << " (";
+		cout << endl << _name << " Test Summary: ";
-		AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount;
+		AnsiColorized(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
-		cout<< " tests skipped)";
+			stats.successCount <<
+			"/" <<
+			stats.testCount;
+		cout << " tests successful";
+		if (stats.skippedCount > 0)
+		{
+			cout << " (";
+			AnsiColorized(cout, formatted, {BOLD, YELLOW}) << stats.skippedCount;
+			cout<< " tests skipped)";
+		}
+		cout << "." << endl << endl;
 	}
-	cout << "." << endl << endl;
-
 	return stats;
 }
 
@@ -372,6 +416,7 @@ int main(int argc, char const *argv[])
 	}
 
 	TestStats global_stats{0, 0};
+	cout << "Running tests..." << endl << endl;
 
 	// Actually run the tests.
 	// Interactive tests are added in InteractiveTests.h
@@ -383,7 +428,14 @@ int main(int argc, char const *argv[])
 		if (ts.smt && options.disableSMT)
 			continue;
 
-		if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion()))
+		auto stats = runTestSuite(
+			ts.testCaseCreator,
+			options,
+			options.testPath / ts.path,
+			ts.subpath,
+			ts.title
+		);
+		if (stats)
 			global_stats += *stats;
 		else
 			return 1;