Adds --test | -t option to isoltest.

Changelog.md

@@ -27,6 +27,8 @@ Bugfixes:
  * Yul: Properly register functions and disallow shadowing between function variables and variables in the outside scope.
 
 
+Build System:
+ * Soltest: Add commandline option `--test` / `-t` to isoltest which takes a string that allows filtering unit tests.
 
 
 ### 0.5.7 (2019-03-26)

test/tools/IsolTestOptions.cpp

@@ -19,9 +19,14 @@
 */
 
 #include <test/tools/IsolTestOptions.h>
+
+#include <libdevcore/Assertions.h>
+
 #include <boost/filesystem.hpp>
-#include <string>
+
 #include <iostream>
+#include <regex>
+#include <string>
 
 namespace fs = boost::filesystem;
 namespace po = boost::program_options;
@@ -32,7 +37,7 @@ namespace test
 {
 
 auto const description = R"(isoltest, tool for interactively managing test contracts.
-Usage: isoltest [Options] --ipcpath ipcpath
+Usage: isoltest [Options]
 Interactively validates test contracts.
 
 Allowed options)";
@@ -51,10 +56,10 @@ IsolTestOptions::IsolTestOptions(std::string* _editor):
 	CommonOptions(description)
 {
 	options.add_options()
+		("editor", po::value<std::string>(_editor)->default_value(editorPath()), "Path to editor for opening test files.")
 		("help", po::bool_switch(&showHelp), "Show this help screen.")
-		("no-color", po::bool_switch(&noColor), "don't use colors")
-		("editor", po::value<std::string>(_editor)->default_value(editorPath()), "editor for opening test files");
-
+		("no-color", po::bool_switch(&noColor), "Don't use colors.")
+		("test,t", po::value<std::string>(&test)->default_value("*/*"), "Filters which test units to include.");
 }
 
 bool IsolTestOptions::parse(int _argc, char const* const* _argv)
@@ -70,5 +75,15 @@ bool IsolTestOptions::parse(int _argc, char const* const* _argv)
 	return res;
 }
 
+void IsolTestOptions::validate() const
+{
+	std::regex filterExpression{"(((\\*+|\\w+|\\w+\\*+)\\/)+(\\*|\\w+\\**))"};
+	assertThrow(
+		regex_match(test, filterExpression),
+		ConfigException,
+		"Invalid test unit filter: " + test
+	);
+}
+
 }
 }

test/tools/IsolTestOptions.h

@@ -30,11 +30,13 @@ namespace test
 
 struct IsolTestOptions: CommonOptions
 {
-	bool noColor = false;
 	bool showHelp = false;
+	bool noColor = false;
+	std::string test = std::string{};
 
 	IsolTestOptions(std::string* _editor);
 	bool parse(int _argc, char const* const* _argv) override;
+	void validate() const override;
 };
 }
 }

test/tools/isoltest.cpp

@@ -32,6 +32,7 @@
 #include <iostream>
 #include <fstream>
 #include <queue>
+#include <regex>
 
 #if defined(_WIN32)
 #include <windows.h>
@@ -60,17 +61,48 @@ struct TestStats
 	}
 };
 
+class TestFilter
+{
+public:
+	explicit TestFilter(string const& _filter): m_filter(_filter)
+	{
+		string filter{m_filter};
+
+		boost::replace_all(filter, "/", "\\/");
+		boost::replace_all(filter, "*", ".*");
+
+		m_filterExpression = regex{"(" + filter + "(\\.sol|\\.yul))"};
+	}
+
+	bool matches(string const& _name) const
+	{
+		return regex_match(_name, m_filterExpression);
+	}
+
+private:
+	string m_filter;
+	regex m_filterExpression;
+};
+
 class TestTool
 {
 public:
 	TestTool(
 		TestCase::TestCaseCreator _testCaseCreator,
+		string const& _filter,
 		string const& _name,
 		fs::path const& _path,
 		string const& _ipcPath,
 		bool _formatted,
 		langutil::EVMVersion _evmVersion
-	): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion)
+	):
+		m_testCaseCreator(_testCaseCreator),
+		m_filter(TestFilter{_filter}),
+		m_name(_name),
+		m_path(_path),
+		m_ipcPath(_ipcPath),
+		m_formatted(_formatted),
+		m_evmVersion(_evmVersion)
 	{}
 
 	enum class Result
@@ -87,6 +119,7 @@ public:
 		TestCase::TestCaseCreator _testCaseCreator,
 		fs::path const& _basepath,
 		fs::path const& _path,
+		string const& _filter,
 		string const& _ipcPath,
 		bool _formatted,
 		langutil::EVMVersion _evmVersion
@@ -104,12 +137,16 @@ private:
 	Request handleResponse(bool _exception);
 
 	TestCase::TestCaseCreator m_testCaseCreator;
+	TestFilter m_filter;
+
 	string const m_name;
 	fs::path const m_path;
-	string m_ipcPath;
+	string const m_ipcPath;
 	bool const m_formatted = false;
 	langutil::EVMVersion const m_evmVersion;
+
 	unique_ptr<TestCase> m_test;
+
 	static bool m_exitRequested;
 };
 
@@ -121,18 +158,23 @@ TestTool::Result TestTool::process()
 	bool success;
 	std::stringstream outputMessages;
 
-	(AnsiColorized(cout, m_formatted, {BOLD}) << m_name << ": ").flush();
-
 	try
 	{
-		m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
-		if (m_test->validateSettings(m_evmVersion))
-			success = m_test->run(outputMessages, "  ", m_formatted);
-		else
+		if (m_filter.matches(m_name))
 		{
-			AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
-			return Result::Skipped;
+			(AnsiColorized(cout, m_formatted, {BOLD}) << m_name << ": ").flush();
+
+			m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
+			if (m_test->validateSettings(m_evmVersion))
+				success = m_test->run(outputMessages, "  ", m_formatted);
+			else
+			{
+				AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
+				return Result::Skipped;
+			}
 		}
+		else
+			return Result::Skipped;
 	}
 	catch(boost::exception const& _e)
 	{
@@ -217,6 +259,7 @@ TestStats TestTool::processPath(
 	TestCase::TestCaseCreator _testCaseCreator,
 	fs::path const& _basepath,
 	fs::path const& _path,
+	string const& _filter,
 	string const& _ipcPath,
 	bool _formatted,
 	langutil::EVMVersion _evmVersion
@@ -251,7 +294,15 @@ TestStats TestTool::processPath(
 		else
 		{
 			++testCount;
-			TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion);
+			TestTool testTool(
+				_testCaseCreator,
+				_filter,
+				currentPath.generic_string(),
+				fullpath,
+				_ipcPath,
+				_formatted,
+				_evmVersion
+			);
 			auto result = testTool.process();
 
 			switch(result)
@@ -317,6 +368,7 @@ boost::optional<TestStats> runTestSuite(
 	string const& _name,
 	fs::path const& _basePath,
 	fs::path const& _subdirectory,
+	string const& _filter,
 	string const& _ipcPath,
 	TestCase::TestCaseCreator _testCaseCreator,
 	bool _formatted,
@@ -331,22 +383,32 @@ boost::optional<TestStats> runTestSuite(
 		return {};
 	}
 
-	TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion);
+	TestStats stats = TestTool::processPath(
+		_testCaseCreator,
+		_basePath,
+		_subdirectory,
+		_filter,
+		_ipcPath,
+		_formatted,
+		_evmVersion
+	);
 
-	cout << endl << _name << " Test Summary: ";
-	AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
-		stats.successCount <<
-		"/" <<
-		stats.testCount;
-	cout << " tests successful";
-	if (stats.skippedCount > 0)
+	if (stats.skippedCount != stats.testCount)
 	{
-		cout << " (";
-		AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount;
-		cout<< " tests skipped)";
+		cout << endl << _name << " Test Summary: ";
+		AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
+			stats.successCount <<
+			"/" <<
+			stats.testCount;
+		cout << " tests successful";
+		if (stats.skippedCount > 0)
+		{
+			cout << " (";
+			AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount;
+			cout<< " tests skipped)";
+		}
+		cout << "." << endl << endl;
 	}
-	cout << "." << endl << endl;
-
 	return stats;
 }
 
@@ -372,6 +434,7 @@ int main(int argc, char const *argv[])
 	}
 
 	TestStats global_stats{0, 0};
+	cout << "Running tests..." << endl << endl;
 
 	// Actually run the tests.
 	// Interactive tests are added in InteractiveTests.h
@@ -383,7 +446,17 @@ int main(int argc, char const *argv[])
 		if (ts.smt && options.disableSMT)
 			continue;
 
-		if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion()))
+		auto stats = runTestSuite(
+			ts.title,
+			options.testPath / ts.path,
+			ts.subpath,
+			options.test,
+			options.ipcPath.string(),
+			ts.testCaseCreator,
+			!options.noColor,
+			options.evmVersion()
+		);
+		if (stats)
 			global_stats += *stats;
 		else
 			return 1;