cerc-io/solidity
10
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #6500 from ethereum/soltest-case-selector

Browse Source 
[soltest] Test filter for isoltest
This commit is contained in:
chriseth 2019-04-17 14:35:52 +02:00 committed by GitHub
commit 2be8f18055
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 136 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Changelog.md
View File

@ -27,6 +27,8 @@ Bugfixes:
* Yul: Properly register functions and disallow shadowing between function variables and variables in the outside scope. * Yul: Properly register functions and disallow shadowing between function variables and variables in the outside scope.
Build System:
* Soltest: Add commandline option `--test` / `-t` to isoltest which takes a string that allows filtering unit tests.
### 0.5.7 (2019-03-26) ### 0.5.7 (2019-03-26)

26
test/tools/IsolTestOptions.cpp
View File

@ -19,9 +19,14 @@
*/ */
#include <test/tools/IsolTestOptions.h> #include <test/tools/IsolTestOptions.h>
#include <libdevcore/Assertions.h>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <string>
#include <iostream> #include <iostream>
#include <regex>
#include <string>
namespace fs = boost::filesystem; namespace fs = boost::filesystem;
namespace po = boost::program_options; namespace po = boost::program_options;
@ -32,7 +37,7 @@ namespace test
{ {
auto const description = R"(isoltest, tool for interactively managing test contracts. auto const description = R"(isoltest, tool for interactively managing test contracts.
Usage: isoltest [Options] --ipcpath ipcpath Usage: isoltest [Options]
Interactively validates test contracts. Interactively validates test contracts.
Allowed options)"; Allowed options)";
@ -51,10 +56,10 @@ IsolTestOptions::IsolTestOptions(std::string* _editor):
CommonOptions(description) CommonOptions(description)
{ {
options.add_options() options.add_options()
("editor", po::value<std::string>(_editor)->default_value(editorPath()), "Path to editor for opening test files.")
("help", po::bool_switch(&showHelp), "Show this help screen.") ("help", po::bool_switch(&showHelp), "Show this help screen.")
("no-color", po::bool_switch(&noColor), "don't use colors") ("no-color", po::bool_switch(&noColor), "Don't use colors.")
("editor", po::value<std::string>(_editor)->default_value(editorPath()), "editor for opening test files"); ("test,t", po::value<std::string>(&testFilter)->default_value("*/*"), "Filters which test units to include.");
} }
bool IsolTestOptions::parse(int _argc, char const* const* _argv) bool IsolTestOptions::parse(int _argc, char const* const* _argv)
@ -70,5 +75,16 @@ bool IsolTestOptions::parse(int _argc, char const* const* _argv)
return res; return res;
} }
void IsolTestOptions::validate() const
{
static std::string filterString{"[a-zA-Z1-9_/*]*"};
static std::regex filterExpression{filterString};
assertThrow(
regex_match(testFilter, filterExpression),
ConfigException,
"Invalid test unit filter - can only contain '" + filterString + ": " + testFilter
);
}
} }
} }

4
test/tools/IsolTestOptions.h
View File

@ -30,11 +30,13 @@ namespace test
struct IsolTestOptions: CommonOptions struct IsolTestOptions: CommonOptions
{ {
bool noColor = false;
bool showHelp = false; bool showHelp = false;
bool noColor = false;
std::string testFilter = std::string{};
IsolTestOptions(std::string* _editor); IsolTestOptions(std::string* _editor);
bool parse(int _argc, char const* const* _argv) override; bool parse(int _argc, char const* const* _argv) override;
void validate() const override;
}; };
} }
} }

146
test/tools/isoltest.cpp
View File

@ -32,6 +32,7 @@
#include <iostream> #include <iostream>
#include <fstream> #include <fstream>
#include <queue> #include <queue>
#include <regex>
#if defined(_WIN32) #if defined(_WIN32)
#include <windows.h> #include <windows.h>
@ -45,6 +46,9 @@ using namespace std;
namespace po = boost::program_options; namespace po = boost::program_options;
namespace fs = boost::filesystem; namespace fs = boost::filesystem;
using TestCreator = TestCase::TestCaseCreator;
using TestOptions = dev::test::IsolTestOptions;
struct TestStats struct TestStats
{ {
int successCount = 0; int successCount = 0;
@ -60,17 +64,43 @@ struct TestStats
} }
}; };
class TestFilter
{
public:
explicit TestFilter(string const& _filter): m_filter(_filter)
{
string filter{m_filter};
boost::replace_all(filter, "/", "\\/");
boost::replace_all(filter, "*", ".*");
m_filterExpression = regex{"(" + filter + "(\\.sol|\\.yul))"};
}
bool matches(string const& _name) const
{
return regex_match(_name, m_filterExpression);
}
private:
string m_filter;
regex m_filterExpression;
};
class TestTool class TestTool
{ {
public: public:
TestTool( TestTool(
TestCase::TestCaseCreator _testCaseCreator, TestCreator _testCaseCreator,
string const& _name, TestOptions const& _options,
fs::path const& _path, fs::path const& _path,
string const& _ipcPath, string const& _name
bool _formatted, ):
langutil::EVMVersion _evmVersion m_testCaseCreator(_testCaseCreator),
): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion) m_options(_options),
m_filter(TestFilter{_options.testFilter}),
m_path(_path),
m_name(_name)
{} {}
enum class Result enum class Result
@ -84,12 +114,10 @@ public:
Result process(); Result process();
static TestStats processPath( static TestStats processPath(
TestCase::TestCaseCreator _testCaseCreator, TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _basepath, fs::path const& _basepath,
fs::path const& _path, fs::path const& _path
string const& _ipcPath,
bool _formatted,
langutil::EVMVersion _evmVersion
); );
static string editor; static string editor;
@ -103,13 +131,14 @@ private:
Request handleResponse(bool _exception); Request handleResponse(bool _exception);
TestCase::TestCaseCreator m_testCaseCreator; TestCreator m_testCaseCreator;
string const m_name; TestOptions const& m_options;
TestFilter m_filter;
fs::path const m_path; fs::path const m_path;
string m_ipcPath; string const m_name;
bool const m_formatted = false;
langutil::EVMVersion const m_evmVersion;
unique_ptr<TestCase> m_test; unique_ptr<TestCase> m_test;
static bool m_exitRequested; static bool m_exitRequested;
}; };
@ -119,52 +148,58 @@ bool TestTool::m_exitRequested = false;
TestTool::Result TestTool::process() TestTool::Result TestTool::process()
{ {
bool success; bool success;
bool formatted{!m_options.noColor};
std::stringstream outputMessages; std::stringstream outputMessages;
(AnsiColorized(cout, m_formatted, {BOLD}) << m_name << ": ").flush();
try try
{ {
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion}); if (m_filter.matches(m_name))
if (m_test->validateSettings(m_evmVersion)) {
success = m_test->run(outputMessages, " ", m_formatted); (AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush();
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
if (m_test->validateSettings(m_options.evmVersion()))
success = m_test->run(outputMessages, " ", formatted);
else else
{ {
AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl; AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
return Result::Skipped; return Result::Skipped;
} }
} }
else
return Result::Skipped;
}
catch(boost::exception const& _e) catch(boost::exception const& _e)
{ {
AnsiColorized(cout, m_formatted, {BOLD, RED}) << AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Exception during test: " << boost::diagnostic_information(_e) << endl; "Exception during test: " << boost::diagnostic_information(_e) << endl;
return Result::Exception; return Result::Exception;
} }
catch (std::exception const& _e) catch (std::exception const& _e)
{ {
AnsiColorized(cout, m_formatted, {BOLD, RED}) << AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Exception during test: " << _e.what() << endl; "Exception during test: " << _e.what() << endl;
return Result::Exception; return Result::Exception;
} }
catch (...) catch (...)
{ {
AnsiColorized(cout, m_formatted, {BOLD, RED}) << AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Unknown exception during test." << endl; "Unknown exception during test." << endl;
return Result::Exception; return Result::Exception;
} }
if (success) if (success)
{ {
AnsiColorized(cout, m_formatted, {BOLD, GREEN}) << "OK" << endl; AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
return Result::Success; return Result::Success;
} }
else else
{ {
AnsiColorized(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl; AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
AnsiColorized(cout, m_formatted, {BOLD, CYAN}) << " Contract:" << endl; AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
m_test->printSource(cout, " ", m_formatted); m_test->printSource(cout, " ", formatted);
m_test->printUpdatedSettings(cout, " ", m_formatted); m_test->printUpdatedSettings(cout, " ", formatted);
cout << endl << outputMessages.str() << endl; cout << endl << outputMessages.str() << endl;
return Result::Failure; return Result::Failure;
@ -214,12 +249,10 @@ TestTool::Request TestTool::handleResponse(bool _exception)
} }
TestStats TestTool::processPath( TestStats TestTool::processPath(
TestCase::TestCaseCreator _testCaseCreator, TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _basepath, fs::path const& _basepath,
fs::path const& _path, fs::path const& _path
string const& _ipcPath,
bool _formatted,
langutil::EVMVersion _evmVersion
) )
{ {
std::queue<fs::path> paths; std::queue<fs::path> paths;
@ -251,7 +284,12 @@ TestStats TestTool::processPath(
else else
{ {
++testCount; ++testCount;
TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion); TestTool testTool(
_testCaseCreator,
_options,
fullpath,
currentPath.string()
);
auto result = testTool.process(); auto result = testTool.process();
switch(result) switch(result)
@ -314,16 +352,15 @@ void setupTerminal()
} }
boost::optional<TestStats> runTestSuite( boost::optional<TestStats> runTestSuite(
string const& _name, TestCreator _testCaseCreator,
TestOptions const& _options,
fs::path const& _basePath, fs::path const& _basePath,
fs::path const& _subdirectory, fs::path const& _subdirectory,
string const& _ipcPath, string const& _name
TestCase::TestCaseCreator _testCaseCreator,
bool _formatted,
langutil::EVMVersion _evmVersion
) )
{ {
fs::path testPath = _basePath / _subdirectory; fs::path testPath{_basePath / _subdirectory};
bool formatted{!_options.noColor};
if (!fs::exists(testPath) || !fs::is_directory(testPath)) if (!fs::exists(testPath) || !fs::is_directory(testPath))
{ {
@ -331,10 +368,17 @@ boost::optional<TestStats> runTestSuite(
return {}; return {};
} }
TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion); TestStats stats = TestTool::processPath(
_testCaseCreator,
_options,
_basePath,
_subdirectory
);
if (stats.skippedCount != stats.testCount)
{
cout << endl << _name << " Test Summary: "; cout << endl << _name << " Test Summary: ";
AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) << AnsiColorized(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
stats.successCount << stats.successCount <<
"/" << "/" <<
stats.testCount; stats.testCount;
@ -342,11 +386,11 @@ boost::optional<TestStats> runTestSuite(
if (stats.skippedCount > 0) if (stats.skippedCount > 0)
{ {
cout << " ("; cout << " (";
AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount; AnsiColorized(cout, formatted, {BOLD, YELLOW}) << stats.skippedCount;
cout<< " tests skipped)"; cout<< " tests skipped)";
} }
cout << "." << endl << endl; cout << "." << endl << endl;
}
return stats; return stats;
} }
@ -372,6 +416,7 @@ int main(int argc, char const *argv[])
} }
TestStats global_stats{0, 0}; TestStats global_stats{0, 0};
cout << "Running tests..." << endl << endl;
// Actually run the tests. // Actually run the tests.
// Interactive tests are added in InteractiveTests.h // Interactive tests are added in InteractiveTests.h
@ -383,7 +428,14 @@ int main(int argc, char const *argv[])
if (ts.smt && options.disableSMT) if (ts.smt && options.disableSMT)
continue; continue;
if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion())) auto stats = runTestSuite(
ts.testCaseCreator,
options,
options.testPath / ts.path,
ts.subpath,
ts.title
);
if (stats)
global_stats += *stats; global_stats += *stats;
else else
return 1; return 1;