solidity/test/tools/isoltest.cpp

489 lines
11 KiB
C++
Raw Normal View History

2018-03-14 18:15:48 +00:00
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
// SPDX-License-Identifier: GPL-3.0
2018-03-14 18:15:48 +00:00
#include <libsolutil/CommonIO.h>
#include <libsolutil/AnsiColorized.h>
2018-10-10 14:12:18 +00:00
#include <memory>
2018-10-10 14:12:18 +00:00
#include <test/Common.h>
#include <test/tools/IsolTestOptions.h>
#include <test/InteractiveTests.h>
2019-07-16 17:05:13 +00:00
#include <test/EVMHost.h>
2018-03-14 18:15:48 +00:00
#include <boost/algorithm/string/replace.hpp>
#include <boost/filesystem.hpp>
#include <cstdlib>
2020-04-01 03:04:29 +00:00
#include <iostream>
2018-03-14 18:15:48 +00:00
#include <queue>
2019-04-10 14:09:32 +00:00
#include <regex>
2020-04-01 03:04:29 +00:00
#include <utility>
2018-03-14 18:15:48 +00:00
#if defined(_WIN32)
#include <windows.h>
#endif
2018-03-14 18:15:48 +00:00
using namespace std;
using namespace solidity;
using namespace solidity::util;
using namespace solidity::frontend;
using namespace solidity::frontend::test;
using namespace solidity::util::formatting;
2018-03-14 18:15:48 +00:00
namespace po = boost::program_options;
namespace fs = boost::filesystem;
using TestCreator = TestCase::TestCaseCreator;
using TestOptions = solidity::test::IsolTestOptions;
struct TestStats
2018-03-14 18:15:48 +00:00
{
int successCount = 0;
int testCount = 0;
int skippedCount = 0;
2019-03-18 13:08:56 +00:00
operator bool() const noexcept { return successCount + skippedCount == testCount; }
2018-10-10 14:12:18 +00:00
TestStats& operator+=(TestStats const& _other) noexcept
2018-10-09 13:36:40 +00:00
{
successCount += _other.successCount;
testCount += _other.testCount;
skippedCount += _other.skippedCount;
2018-10-09 13:36:40 +00:00
return *this;
}
2018-03-14 18:15:48 +00:00
};
2019-04-10 14:09:32 +00:00
class TestFilter
{
public:
2020-04-01 03:04:29 +00:00
explicit TestFilter(string _filter): m_filter(std::move(_filter))
2019-04-10 14:09:32 +00:00
{
string filter{m_filter};
boost::replace_all(filter, "/", "\\/");
boost::replace_all(filter, "*", ".*");
m_filterExpression = regex{"(" + filter + "(\\.sol|\\.yul))"};
}
bool matches(fs::path const& _path, string const& _name) const
2019-04-10 14:09:32 +00:00
{
return regex_match(_name, m_filterExpression) && solidity::test::isValidSemanticTestPath(_path);
2019-04-10 14:09:32 +00:00
}
private:
string m_filter;
regex m_filterExpression;
};
class TestTool
2018-03-14 18:15:48 +00:00
{
public:
TestTool(
TestCreator _testCaseCreator,
TestOptions const& _options,
2020-04-01 03:04:29 +00:00
fs::path _path,
string _name
2019-04-10 14:09:32 +00:00
):
m_testCaseCreator(_testCaseCreator),
m_options(_options),
2019-04-17 09:45:51 +00:00
m_filter(TestFilter{_options.testFilter}),
2020-04-01 03:04:29 +00:00
m_path(std::move(_path)),
m_name(std::move(_name))
2018-03-14 18:15:48 +00:00
{}
enum class Result
{
Success,
Failure,
Exception,
Skipped
2018-03-14 18:15:48 +00:00
};
Result process();
static TestStats processPath(
TestCreator _testCaseCreator,
TestOptions const& _options,
2018-03-14 18:15:48 +00:00
fs::path const& _basepath,
fs::path const& _path
2018-03-14 18:15:48 +00:00
);
private:
enum class Request
{
Skip,
Rerun,
Quit
};
2021-03-12 22:43:50 +00:00
void updateTestCase();
2019-03-18 13:08:56 +00:00
Request handleResponse(bool _exception);
2018-03-14 18:15:48 +00:00
TestCreator m_testCaseCreator;
TestOptions const& m_options;
2019-04-10 14:09:32 +00:00
TestFilter m_filter;
2018-03-14 18:15:48 +00:00
fs::path const m_path;
string const m_name;
2019-04-10 14:09:32 +00:00
unique_ptr<TestCase> m_test;
2019-04-10 14:09:32 +00:00
static bool m_exitRequested;
2018-03-14 18:15:48 +00:00
};
bool TestTool::m_exitRequested = false;
2018-03-14 18:15:48 +00:00
TestTool::Result TestTool::process()
2018-03-14 18:15:48 +00:00
{
bool formatted{!m_options.noColor};
2018-03-14 18:15:48 +00:00
try
{
if (m_filter.matches(m_path, m_name))
{
(AnsiColorized(cout, formatted, {BOLD}) << m_name << ": ").flush();
2019-04-10 14:09:32 +00:00
m_test = m_testCaseCreator(TestCase::Config{
m_path.string(),
m_options.evmVersion(),
2020-06-05 23:10:48 +00:00
m_options.vmPaths,
m_options.enforceViaYul,
m_options.enforceCompileToEwasm,
m_options.enforceGasTest,
m_options.enforceGasTestMinValue
});
2020-02-18 16:13:13 +00:00
if (m_test->shouldRun())
2020-11-25 01:11:11 +00:00
{
std::stringstream outputMessages;
switch (TestCase::TestResult result = m_test->run(outputMessages, " ", formatted))
{
case TestCase::TestResult::Success:
AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
return Result::Success;
default:
AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
m_test->printSource(cout, " ", formatted);
m_test->printSettings(cout, " ", formatted);
cout << endl << outputMessages.str() << endl;
return result == TestCase::TestResult::FatalError ? Result::Exception : Result::Failure;
}
2020-11-25 01:11:11 +00:00
}
2019-04-10 14:09:32 +00:00
else
{
AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
2019-04-10 14:09:32 +00:00
return Result::Skipped;
}
}
2019-04-10 14:09:32 +00:00
else
return Result::Skipped;
2018-03-14 18:15:48 +00:00
}
2019-05-29 20:26:18 +00:00
catch (boost::exception const& _e)
2018-03-14 18:15:48 +00:00
{
AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Exception during test: " << boost::diagnostic_information(_e) << endl;
return Result::Exception;
}
catch (std::exception const& _e)
{
AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Exception during test: " << boost::diagnostic_information(_e) << endl;
return Result::Exception;
}
catch (...)
{
AnsiColorized(cout, formatted, {BOLD, RED}) <<
"Unknown exception during test: " << boost::current_exception_diagnostic_information() << endl;
return Result::Exception;
2018-03-14 18:15:48 +00:00
}
}
2021-03-12 22:43:50 +00:00
void TestTool::updateTestCase()
{
ofstream file(m_path.string(), ios::trunc);
m_test->printSource(file);
m_test->printUpdatedSettings(file);
file << "// ----" << endl;
m_test->printUpdatedExpectations(file, "// ");
}
2019-03-18 13:08:56 +00:00
TestTool::Request TestTool::handleResponse(bool _exception)
2018-03-14 18:15:48 +00:00
{
2021-03-12 22:43:50 +00:00
if (!_exception && m_options.acceptUpdates)
{
updateTestCase();
return Request::Rerun;
}
if (_exception)
2018-03-14 18:15:48 +00:00
cout << "(e)dit/(s)kip/(q)uit? ";
else
cout << "(e)dit/(u)pdate expectations/(s)kip/(q)uit? ";
cout.flush();
while (true)
{
switch(readStandardInputChar())
{
case 's':
cout << endl;
return Request::Skip;
case 'u':
if (_exception)
2018-03-14 18:15:48 +00:00
break;
else
{
cout << endl;
2021-03-12 22:43:50 +00:00
updateTestCase();
2018-03-14 18:15:48 +00:00
return Request::Rerun;
}
case 'e':
cout << endl << endl;
if (system((m_options.editor + " \"" + m_path.string() + "\"").c_str()))
2018-03-14 18:15:48 +00:00
cerr << "Error running editor command." << endl << endl;
return Request::Rerun;
case 'q':
cout << endl;
return Request::Quit;
default:
break;
}
}
}
TestStats TestTool::processPath(
TestCreator _testCaseCreator,
TestOptions const& _options,
2018-03-14 18:15:48 +00:00
fs::path const& _basepath,
fs::path const& _path
2018-03-14 18:15:48 +00:00
)
{
std::queue<fs::path> paths;
paths.push(_path);
int successCount = 0;
int testCount = 0;
int skippedCount = 0;
2018-03-14 18:15:48 +00:00
while (!paths.empty())
{
auto currentPath = paths.front();
fs::path fullpath = _basepath / currentPath;
if (fs::is_directory(fullpath))
{
paths.pop();
for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
fs::directory_iterator(fullpath),
fs::directory_iterator()
))
if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
paths.push(currentPath / entry.path().filename());
2018-03-14 18:15:48 +00:00
}
else if (m_exitRequested)
{
++testCount;
paths.pop();
}
2018-03-14 18:15:48 +00:00
else
{
++testCount;
2019-04-10 14:09:32 +00:00
TestTool testTool(
_testCaseCreator,
_options,
2019-04-10 14:09:32 +00:00
fullpath,
currentPath.generic_path().string()
2019-04-10 14:09:32 +00:00
);
2018-03-14 18:15:48 +00:00
auto result = testTool.process();
switch(result)
{
case Result::Failure:
case Result::Exception:
switch(testTool.handleResponse(result == Result::Exception))
2018-03-14 18:15:48 +00:00
{
case Request::Quit:
paths.pop();
m_exitRequested = true;
break;
2018-03-14 18:15:48 +00:00
case Request::Rerun:
cout << "Re-running test case..." << endl;
--testCount;
2018-03-14 18:15:48 +00:00
break;
case Request::Skip:
paths.pop();
++skippedCount;
2018-03-14 18:15:48 +00:00
break;
}
break;
case Result::Success:
paths.pop();
++successCount;
break;
case Result::Skipped:
paths.pop();
++skippedCount;
break;
2018-03-14 18:15:48 +00:00
}
}
}
return { successCount, testCount, skippedCount };
2018-03-14 18:15:48 +00:00
}
2018-10-09 13:25:28 +00:00
namespace
{
void setupTerminal()
{
#if defined(_WIN32) && defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
// Set output mode to handle virtual terminal (ANSI escape sequences)
// ignore any error, as this is just a "nice-to-have"
// only windows needs to be taken care of, as other platforms (Linux/OSX) support them natively.
HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE);
if (hOut == INVALID_HANDLE_VALUE)
return;
DWORD dwMode = 0;
if (!GetConsoleMode(hOut, &dwMode))
return;
dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
if (!SetConsoleMode(hOut, dwMode))
return;
#endif
}
std::optional<TestStats> runTestSuite(
TestCreator _testCaseCreator,
TestOptions const& _options,
2018-10-09 13:36:40 +00:00
fs::path const& _basePath,
fs::path const& _subdirectory,
string const& _name
2018-10-09 13:36:40 +00:00
)
{
fs::path testPath{_basePath / _subdirectory};
bool formatted{!_options.noColor};
2018-10-09 13:36:40 +00:00
if (!fs::exists(testPath) || !fs::is_directory(testPath))
{
cerr << _name << " tests not found. Use the --testpath argument." << endl;
return std::nullopt;
2018-10-09 13:36:40 +00:00
}
2019-04-10 14:09:32 +00:00
TestStats stats = TestTool::processPath(
_testCaseCreator,
_options,
2019-04-10 14:09:32 +00:00
_basePath,
_subdirectory
2019-04-10 14:09:32 +00:00
);
2018-10-09 13:36:40 +00:00
2019-04-10 14:09:32 +00:00
if (stats.skippedCount != stats.testCount)
{
2019-04-10 14:09:32 +00:00
cout << endl << _name << " Test Summary: ";
AnsiColorized(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
2019-04-10 14:09:32 +00:00
stats.successCount <<
"/" <<
stats.testCount;
cout << " tests successful";
if (stats.skippedCount > 0)
{
cout << " (";
AnsiColorized(cout, formatted, {BOLD, YELLOW}) << stats.skippedCount;
2019-04-10 14:09:32 +00:00
cout<< " tests skipped)";
}
cout << "." << endl << endl;
}
2018-10-09 13:36:40 +00:00
return stats;
}
2018-10-09 13:25:28 +00:00
}
int main(int argc, char const *argv[])
2018-03-14 18:15:48 +00:00
{
try
2018-03-14 18:15:48 +00:00
{
setupTerminal();
{
auto options = std::make_unique<solidity::test::IsolTestOptions>();
if (!options->parse(argc, argv))
return -1;
options->validate();
solidity::test::CommonOptions::setSingleton(std::move(options));
}
auto& options = dynamic_cast<solidity::test::IsolTestOptions const&>(solidity::test::CommonOptions::get());
if (!solidity::test::loadVMs(options))
return 1;
if (options.disableSemanticTests)
cout << endl << "--- SKIPPING ALL SEMANTICS TESTS ---" << endl << endl;
2018-03-14 18:15:48 +00:00
TestStats global_stats{0, 0};
cout << "Running tests..." << endl << endl;
// Actually run the tests.
// Interactive tests are added in InteractiveTests.h
for (auto const& ts: g_interactiveTestsuites)
{
if (ts.needsVM && options.disableSemanticTests)
continue;
if (ts.smt && options.disableSMT)
continue;
auto stats = runTestSuite(
ts.testCaseCreator,
options,
options.testPath / ts.path,
ts.subpath,
ts.title
);
if (stats)
global_stats += *stats;
else
return 1;
}
2019-07-16 17:05:13 +00:00
cout << endl << "Summary: ";
AnsiColorized(cout, !options.noColor, {BOLD, global_stats ? GREEN : RED}) <<
global_stats.successCount << "/" << global_stats.testCount;
cout << " tests successful";
if (global_stats.skippedCount > 0)
{
cout << " (";
AnsiColorized(cout, !options.noColor, {BOLD, YELLOW}) << global_stats.skippedCount;
cout << " tests skipped)";
}
cout << "." << endl;
2020-06-05 23:10:48 +00:00
if (options.disableSemanticTests)
cout << "\nNOTE: Skipped semantics tests.\n" << endl;
2018-03-14 18:15:48 +00:00
return global_stats ? 0 : 1;
}
catch (std::exception const& _exception)
{
cerr << _exception.what() << endl;
return 1;
}
2018-03-14 18:15:48 +00:00
}