cerc-io/solidity
16
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #6500 from ethereum/soltest-case-selector

Browse Source 
[soltest] Test filter for isoltest
This commit is contained in:
chriseth 2019-04-17 14:35:52 +02:00 committed by GitHub
commit 2be8f18055
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 136 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Changelog.md
View File

@ -27,6 +27,8 @@ Bugfixes:
* Yul: Properly register functions and disallow shadowing between function variables and variables in the outside scope.
Build System:
* Soltest: Add commandline option `--test` / `-t` to isoltest which takes a string that allows filtering unit tests.
### 0.5.7 (2019-03-26)

26
test/tools/IsolTestOptions.cpp
View File

@ -19,9 +19,14 @@
*/
#include <test/tools/IsolTestOptions.h>
#include <libdevcore/Assertions.h>
#include <boost/filesystem.hpp>
#include <string>
#include <iostream>
#include <regex>
#include <string>
namespace fs = boost::filesystem;
namespace po = boost::program_options;
@ -32,7 +37,7 @@ namespace test
{
auto const description = R"(isoltest, tool for interactively managing test contracts.
Usage: isoltest [Options] --ipcpath ipcpath
Usage: isoltest [Options]
Interactively validates test contracts.
Allowed options)";
@ -51,10 +56,10 @@ IsolTestOptions::IsolTestOptions(std::string* _editor):
CommonOptions(description)
{
options.add_options()
("editor", po::value<std::string>(_editor)->default_value(editorPath()), "Path to editor for opening test files.")
("help", po::bool_switch(&showHelp), "Show this help screen.")
("no-color", po::bool_switch(&noColor), "don't use colors")
("editor", po::value<std::string>(_editor)->default_value(editorPath()), "editor for opening test files");
("no-color", po::bool_switch(&noColor), "Don't use colors.")
("test,t", po::value<std::string>(&testFilter)->default_value("*/*"), "Filters which test units to include.");
}
bool IsolTestOptions::parse(int _argc, char const* const* _argv)
@ -70,5 +75,16 @@ bool IsolTestOptions::parse(int _argc, char const* const* _argv)
return res;
}
void IsolTestOptions::validate() const
{
static std::string filterString{"[a-zA-Z1-9_/*]*"};
static std::regex filterExpression{filterString};
assertThrow(
regex_match(testFilter, filterExpression),
ConfigException,
"Invalid test unit filter - can only contain '" + filterString + ": " + testFilter
);
}
}
}

4
test/tools/IsolTestOptions.h
View File

@ -30,11 +30,13 @@ namespace test
struct IsolTestOptions: CommonOptions
{
bool noColor = false;
bool showHelp = false;
bool noColor = false;
std::string testFilter = std::string{};
IsolTestOptions(std::string* _editor);
bool parse(int _argc, char const* const* _argv) override;
void validate() const override;
};
}
}

168
test/tools/isoltest.cpp
View File

@ -32,6 +32,7 @@
#include <iostream>
#include <fstream>
#include <queue>
#include <regex>
#if defined(_WIN32)
#include <windows.h>
@ -45,6 +46,9 @@ using namespace std;
namespace po = boost::program_options;
namespace fs = boost::filesystem;
using TestCreator = TestCase::TestCaseCreator;
using TestOptions = dev::test::IsolTestOptions;
struct TestStats
{
int successCount = 0;
@ -60,17 +64,43 @@ struct TestStats
}
};
class TestFilter
{
public:
explicit TestFilter(string const& _filter): m_filter(_filter)
{
string filter{m_filter};
boost::replace_all(filter, "/", "\\/");
boost::replace_all(filter, "*", ".*");
m_filterExpression = regex{"(" + filter + "(\\.sol|\\.yul))"};
}
bool matches(string const& _name) const
{
return regex_match(_name, m_filterExpression);
}
private:
string m_filter;
regex m_filterExpression;
};
class TestTool
{
public:
TestTool(
TestCase::TestCaseCreator _testCaseCreator,
string const& _name,
TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _path,
string const& _ipcPath,
bool _formatted,
langutil::EVMVersion _evmVersion
): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion)
string const& _name
):
m_testCaseCreator(_testCaseCreator),
m_options(_options),
m_filter(TestFilter{_options.testFilter}),
m_path(_path),
m_name(_name)
{}
enum class Result
@ -84,12 +114,10 @@ public:
Result process();
static TestStats processPath(
TestCase::TestCaseCreator _testCaseCreator,
TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _basepath,
fs::path const& _path,
string const& _ipcPath,
bool _formatted,
langutil::EVMVersion _evmVersion
fs::path const& _path
);
static string editor;
@ -103,13 +131,14 @@ private:
Request handleResponse(bool _exception);
TestCase::TestCaseCreator m_testCaseCreator;
string const m_name;
TestCreator m_testCaseCreator;
TestOptions const& m_options;
TestFilter m_filter;
fs::path const m_path;
string m_ipcPath;
bool const m_formatted = false;
langutil::EVMVersion const m_evmVersion;
string const m_name;
unique_ptr<TestCase> m_test;
static bool m_exitRequested;
};
@ -119,52 +148,58 @@ bool TestTool::m_exitRequested = false;
TestTool::Result TestTool::process()
{
bool success;
bool formatted{!m_options.noColor};
std::stringstream outputMessages;
(AnsiColorized(cout, m_formatted, {BOLD}) << m_name << ": ").flush();
try
{
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
if (m_test->validateSettings(m_evmVersion))
success = m_test->run(outputMessages, " ", m_formatted);
else
if (m_filter.matches(m_name))
{
AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
return Result::Skipped;
(AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush();
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
if (m_test->validateSettings(m_options.evmVersion()))
success = m_test->run(outputMessages, " ", formatted);
else
{
AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
return Result::Skipped;
}
}
else
return Result::Skipped;
}
catch(boost::exception const& _e)
{
AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Exception during test: " << boost::diagnostic_information(_e) << endl;
return Result::Exception;
}
catch (std::exception const& _e)
{
AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Exception during test: " << _e.what() << endl;
return Result::Exception;
}
catch (...)
{
AnsiColorized(cout, m_formatted, {BOLD, RED}) <<
AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Unknown exception during test." << endl;
return Result::Exception;
}
if (success)
{
AnsiColorized(cout, m_formatted, {BOLD, GREEN}) << "OK" << endl;
AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
return Result::Success;
}
else
{
AnsiColorized(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl;
AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
AnsiColorized(cout, m_formatted, {BOLD, CYAN}) << " Contract:" << endl;
m_test->printSource(cout, " ", m_formatted);
m_test->printUpdatedSettings(cout, " ", m_formatted);
AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
m_test->printSource(cout, " ", formatted);
m_test->printUpdatedSettings(cout, " ", formatted);
cout << endl << outputMessages.str() << endl;
return Result::Failure;
@ -214,12 +249,10 @@ TestTool::Request TestTool::handleResponse(bool _exception)
}
TestStats TestTool::processPath(
TestCase::TestCaseCreator _testCaseCreator,
TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _basepath,
fs::path const& _path,
string const& _ipcPath,
bool _formatted,
langutil::EVMVersion _evmVersion
fs::path const& _path
)
{
std::queue<fs::path> paths;
@ -251,7 +284,12 @@ TestStats TestTool::processPath(
else
{
++testCount;
TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion);
TestTool testTool(
_testCaseCreator,
_options,
fullpath,
currentPath.string()
);
auto result = testTool.process();
switch(result)
@ -314,16 +352,15 @@ void setupTerminal()
}
boost::optional<TestStats> runTestSuite(
string const& _name,
TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _basePath,
fs::path const& _subdirectory,
string const& _ipcPath,
TestCase::TestCaseCreator _testCaseCreator,
bool _formatted,
langutil::EVMVersion _evmVersion
string const& _name
)
{
fs::path testPath = _basePath / _subdirectory;
fs::path testPath{_basePath / _subdirectory};
bool formatted{!_options.noColor};
if (!fs::exists(testPath) || !fs::is_directory(testPath))
{
@ -331,22 +368,29 @@ boost::optional<TestStats> runTestSuite(
return {};
}
TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion);
TestStats stats = TestTool::processPath(
_testCaseCreator,
_options,
_basePath,
_subdirectory
);
cout << endl << _name << " Test Summary: ";
AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
stats.successCount <<
"/" <<
stats.testCount;
cout << " tests successful";
if (stats.skippedCount > 0)
if (stats.skippedCount != stats.testCount)
{
cout << " (";
AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount;
cout<< " tests skipped)";
cout << endl << _name << " Test Summary: ";
AnsiColorized(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
stats.successCount <<
"/" <<
stats.testCount;
cout << " tests successful";
if (stats.skippedCount > 0)
{
cout << " (";
AnsiColorized(cout, formatted, {BOLD, YELLOW}) << stats.skippedCount;
cout<< " tests skipped)";
}
cout << "." << endl << endl;
}
cout << "." << endl << endl;
return stats;
}
@ -372,6 +416,7 @@ int main(int argc, char const *argv[])
}
TestStats global_stats{0, 0};
cout << "Running tests..." << endl << endl;
// Actually run the tests.
// Interactive tests are added in InteractiveTests.h
@ -383,7 +428,14 @@ int main(int argc, char const *argv[])
if (ts.smt && options.disableSMT)
continue;
if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion()))
auto stats = runTestSuite(
ts.testCaseCreator,
options,
options.testPath / ts.path,
ts.subpath,
ts.title
);
if (stats)
global_stats += *stats;
else
return 1;