cerc-io/solidity
16
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #6300 from ethereum/extractedTestsEVMVersionRules

Browse Source 
Support EVM Version rules for extracted tests.
This commit is contained in:
chriseth 2019-03-18 15:29:04 +01:00 committed by GitHub
commit 304ef77b4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 169 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
test/Common.cpp
View File

@ -75,6 +75,7 @@ CommonOptions::CommonOptions(std::string _caption):
)
{
options.add_options()
("evm-version", po::value(&evmVersionString), "which evm version to use")
("testpath", po::value<fs::path>(&this->testPath)->default_value(dev::test::testPath()), "path to test files")
("ipcpath", po::value<fs::path>(&ipcPath)->default_value(IPCEnvOrDefaultPath()), "path to ipc socket")
("no-ipc", po::bool_switch(&disableIPC), "disable semantic tests")
@ -121,6 +122,20 @@ bool CommonOptions::parse(int argc, char const* const* argv)
return true;
}
langutil::EVMVersion CommonOptions::evmVersion() const
{
if (!evmVersionString.empty())
{
auto version = langutil::EVMVersion::fromString(evmVersionString);
if (!version)
throw std::runtime_error("Invalid EVM version: " + evmVersionString);
return *version;
}
else
return langutil::EVMVersion();
}
}
}

6
test/Common.h
View File

@ -18,6 +18,7 @@
#pragma once
#include <libdevcore/Exceptions.h>
#include <liblangutil/EVMVersion.h>
#include <boost/filesystem/path.hpp>
#include <boost/program_options.hpp>
@ -39,6 +40,8 @@ struct CommonOptions: boost::noncopyable
bool disableIPC = false;
bool disableSMT = false;
langutil::EVMVersion evmVersion() const;
virtual bool parse(int argc, char const* const* argv);
// Throws a ConfigException on error
virtual void validate() const;
@ -47,6 +50,9 @@ protected:
CommonOptions(std::string caption = "");
boost::program_options::options_description options;
private:
std::string evmVersionString;
};
}

6
test/ExecutionFramework.cpp
View File

@ -50,13 +50,13 @@ string getIPCSocketPath()
}
ExecutionFramework::ExecutionFramework():
ExecutionFramework(getIPCSocketPath())
ExecutionFramework(getIPCSocketPath(), dev::test::Options::get().evmVersion())
{
}
ExecutionFramework::ExecutionFramework(string const& _ipcPath):
ExecutionFramework::ExecutionFramework(string const& _ipcPath, langutil::EVMVersion _evmVersion):
m_rpc(RPCSession::instance(_ipcPath)),
m_evmVersion(dev::test::Options::get().evmVersion()),
m_evmVersion(_evmVersion),
m_optimize(dev::test::Options::get().optimize),
m_showMessages(dev::test::Options::get().showMessages),
m_sender(m_rpc.account(0))

2
test/ExecutionFramework.h
View File

@ -53,7 +53,7 @@ class ExecutionFramework
public:
ExecutionFramework();
explicit ExecutionFramework(std::string const& _ipcPath);
explicit ExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
virtual ~ExecutionFramework() = default;
virtual bytes const& compileAndRunWithoutCheck(

15
test/Options.cpp
View File

@ -53,22 +53,7 @@ Options::Options()
options.add_options()
("optimize", po::bool_switch(&optimize), "enables optimization")
("abiencoderv2", po::bool_switch(&useABIEncoderV2), "enables abi encoder v2")
("evm-version", po::value(&evmVersionString), "which evm version to use")
("show-messages", po::bool_switch(&showMessages), "enables message output");
parse(suite.argc, suite.argv);
}
langutil::EVMVersion Options::evmVersion() const
{
if (!evmVersionString.empty())
{
// We do this check as opposed to in the constructor because the BOOST_REQUIRE
// macros cannot yet be used in the constructor.
auto version = langutil::EVMVersion::fromString(evmVersionString);
BOOST_REQUIRE_MESSAGE(version, "Invalid EVM version: " + evmVersionString);
return *version;
}
else
return langutil::EVMVersion();
}

4
test/Options.h
View File

@ -37,13 +37,9 @@ struct Options: CommonOptions
bool showMessages = false;
bool useABIEncoderV2 = false;
langutil::EVMVersion evmVersion() const;
static Options const& get();
private:
std::string evmVersionString;
Options();
};

44
test/TestCase.cpp
View File

@ -17,6 +17,7 @@
#include <test/TestCase.h>
#include <boost/algorithm/cxx11/none_of.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/predicate.hpp>
@ -35,16 +36,57 @@ bool TestCase::isTestFilename(boost::filesystem::path const& _filename)
!boost::starts_with(_filename.string(), ".");
}
bool TestCase::supportedForEVMVersion(langutil::EVMVersion _evmVersion) const
{
return boost::algorithm::none_of(m_evmVersionRules, [&](auto const& rule) { return !rule(_evmVersion); });
}
string TestCase::parseSource(istream& _stream)
{
string source;
string line;
string const delimiter("// ----");
static string const delimiter("// ----");
static string const evmVersion("// EVMVersion: ");
bool isTop = true;
while (getline(_stream, line))
if (boost::algorithm::starts_with(line, delimiter))
break;
else
{
if (isTop && boost::algorithm::starts_with(line, evmVersion))
{
string versionString = line.substr(evmVersion.size() + 1);
auto version = langutil::EVMVersion::fromString(versionString);
if (!version)
throw runtime_error("Invalid EVM version: \"" + versionString + "\"");
switch (line.at(evmVersion.size()))
{
case '>':
m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
return version < _version;
});
break;
case '<':
m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
return _version < version;
});
break;
case '=':
m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
return _version == version;
});
break;
case '!':
m_evmVersionRules.emplace_back([version](langutil::EVMVersion _version) {
return !(_version == version);
});
break;
}
}
else
isTop = false;
source += line + "\n";
}
return source;
}

13
test/TestCase.h
View File

@ -17,11 +17,15 @@
#pragma once
#include <liblangutil/EVMVersion.h>
#include <boost/filesystem.hpp>
#include <functional>
#include <iosfwd>
#include <memory>
#include <string>
#include <vector>
namespace dev
{
@ -46,6 +50,7 @@ public:
{
std::string filename;
std::string ipcPath;
langutil::EVMVersion evmVersion;
};
using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
@ -69,8 +74,11 @@ public:
static bool isTestFilename(boost::filesystem::path const& _filename);
/// Returns true, if the test case is supported for EVM version @arg _evmVersion, false otherwise.
bool supportedForEVMVersion(langutil::EVMVersion _evmVersion) const;
protected:
static std::string parseSource(std::istream& _file);
std::string parseSource(std::istream& _file);
static void expect(std::string::iterator& _it, std::string::iterator _end, std::string::value_type _c);
template<typename IteratorType>
@ -86,7 +94,8 @@ protected:
while (_it != _end && *_it == '/')
++_it;
}
private:
std::vector<std::function<bool(langutil::EVMVersion)>> m_evmVersionRules;
};
}

8
test/boostTest.cpp
View File

@ -80,7 +80,7 @@ int registerTests(
{
int numTestsAdded = 0;
fs::path fullpath = _basepath / _path;
TestCase::Config config{fullpath.string(), _ipcPath};
TestCase::Config config{fullpath.string(), _ipcPath, dev::test::Options::get().evmVersion()};
if (fs::is_directory(fullpath))
{
test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
@ -104,8 +104,10 @@ int registerTests(
try
{
stringstream errorStream;
if (!_testCaseCreator(config)->run(errorStream))
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
auto testCase = _testCaseCreator(config);
if (testCase->supportedForEVMVersion(dev::test::Options::get().evmVersion()))
if (!testCase->run(errorStream))
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
}
catch (boost::exception const& _e)
{

6
test/libsolidity/SMTCheckerJSONTest.cpp
View File

@ -35,8 +35,8 @@ using namespace dev;
using namespace std;
using namespace boost::unit_test;
SMTCheckerTest::SMTCheckerTest(string const& _filename)
: SyntaxTest(_filename)
SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _evmVersion)
: SyntaxTest(_filename, _evmVersion)
{
if (!boost::algorithm::ends_with(_filename, ".sol"))
BOOST_THROW_EXCEPTION(runtime_error("Invalid test contract file name: \"" + _filename + "\"."));
@ -49,7 +49,7 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename)
BOOST_THROW_EXCEPTION(runtime_error("Invalid JSON file."));
}
bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
StandardCompiler compiler;

6
test/libsolidity/SMTCheckerJSONTest.h
View File

@ -35,11 +35,11 @@ class SMTCheckerTest: public SyntaxTest
public:
static std::unique_ptr<TestCase> create(Config const& _config)
{
return std::unique_ptr<TestCase>(new SMTCheckerTest(_config.filename));
return std::make_unique<SMTCheckerTest>(_config.filename, _config.evmVersion);
}
SMTCheckerTest(std::string const& _filename);
SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
private:
std::vector<std::string> hashesFromJson(Json::Value const& _jsonObj, std::string const& _auxInput, std::string const& _smtlib);

8
test/libsolidity/SemanticTest.cpp
View File

@ -36,8 +36,8 @@ using namespace boost::unit_test;
namespace fs = boost::filesystem;
SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
SolidityExecutionFramework(_ipcPath)
SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, langutil::EVMVersion _evmVersion):
SolidityExecutionFramework(_ipcPath, _evmVersion)
{
ifstream file(_filename);
soltestAssert(file, "Cannot open test contract: \"" + _filename + "\".");
@ -47,7 +47,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath):
parseExpectations(file);
}
bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
@ -87,7 +87,7 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _
return true;
}
void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const
{
stringstream stream(m_source);
string line;

8
test/libsolidity/SemanticTest.h
View File

@ -44,12 +44,12 @@ class SemanticTest: public SolidityExecutionFramework, public TestCase
{
public:
static std::unique_ptr<TestCase> create(Config const& _options)
{ return std::make_unique<SemanticTest>(_options.filename, _options.ipcPath); }
{ return std::make_unique<SemanticTest>(_options.filename, _options.ipcPath, _options.evmVersion); }
explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath);
explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;
/// Instantiates a test file parser that parses the additional comment section at the end of

4
test/libsolidity/SolidityExecutionFramework.cpp
View File

@ -33,7 +33,7 @@ SolidityExecutionFramework::SolidityExecutionFramework():
{
}
SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath):
ExecutionFramework(_ipcPath)
SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion _evmVersion):
ExecutionFramework(_ipcPath, _evmVersion)
{
}

2
test/libsolidity/SolidityExecutionFramework.h
View File

@ -43,7 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework
public:
SolidityExecutionFramework();
SolidityExecutionFramework(std::string const& _ipcPath);
SolidityExecutionFramework(std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
virtual bytes const& compileAndRunWithoutCheck(
std::string const& _sourceCode,

12
test/libsolidity/SyntaxTest.cpp
View File

@ -52,7 +52,7 @@ int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
}
SyntaxTest::SyntaxTest(string const& _filename)
SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion): m_evmVersion(_evmVersion)
{
ifstream file(_filename);
if (!file)
@ -63,12 +63,12 @@ SyntaxTest::SyntaxTest(string const& _filename)
m_expectations = parseExpectations(file);
}
bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
string const versionPragma = "pragma solidity >=0.0;\n";
m_compiler.reset();
m_compiler.addSource("", versionPragma + m_source);
m_compiler.setEVMVersion(dev::test::Options::get().evmVersion());
m_compiler.setEVMVersion(m_evmVersion);
if (m_compiler.parse())
m_compiler.analyze();
@ -95,7 +95,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _fo
return printExpectationAndError(_stream, _linePrefix, _formatted);
}
bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool const _formatted)
bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool _formatted)
{
if (m_expectations != m_errorList)
{
@ -109,7 +109,7 @@ bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _lineP
return true;
}
void SyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool const _formatted) const
void SyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const
{
if (_formatted)
{
@ -162,7 +162,7 @@ void SyntaxTest::printErrorList(
ostream& _stream,
vector<SyntaxTestError> const& _errorList,
string const& _linePrefix,
bool const _formatted
bool _formatted
)
{
if (_errorList.empty())

13
test/libsolidity/SyntaxTest.h
View File

@ -54,12 +54,12 @@ class SyntaxTest: AnalysisFramework, public TestCase
{
public:
static std::unique_ptr<TestCase> create(Config const& _config)
{ return std::unique_ptr<TestCase>(new SyntaxTest(_config.filename)); }
SyntaxTest(std::string const& _filename);
{ return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion); }
SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override
{
if (!m_errorList.empty())
@ -72,16 +72,17 @@ protected:
std::ostream& _stream,
std::vector<SyntaxTestError> const& _errors,
std::string const& _linePrefix,
bool const _formatted = false
bool _formatted = false
);
virtual bool printExpectationAndError(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false);
virtual bool printExpectationAndError(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false);
static std::vector<SyntaxTestError> parseExpectations(std::istream& _stream);
std::string m_source;
std::vector<SyntaxTestError> m_expectations;
std::vector<SyntaxTestError> m_errorList;
langutil::EVMVersion const m_evmVersion;
};
}

2
test/tools/IsolTestOptions.h
View File

@ -19,6 +19,8 @@
#pragma once
#include <liblangutil/EVMVersion.h>
#include <test/Common.h>
namespace dev

72
test/tools/isoltest.cpp
View File

@ -47,13 +47,15 @@ namespace fs = boost::filesystem;
struct TestStats
{
int successCount;
int testCount;
operator bool() const { return successCount == testCount; }
int successCount = 0;
int testCount = 0;
int skippedCount = 0;
operator bool() const noexcept { return successCount + skippedCount == testCount; }
TestStats& operator+=(TestStats const& _other) noexcept
{
successCount += _other.successCount;
testCount += _other.testCount;
skippedCount += _other.skippedCount;
return *this;
}
};
@ -66,15 +68,17 @@ public:
string const& _name,
fs::path const& _path,
string const& _ipcPath,
bool _formatted
): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted)
bool _formatted,
langutil::EVMVersion _evmVersion
): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted), m_evmVersion(_evmVersion)
{}
enum class Result
{
Success,
Failure,
Exception
Exception,
Skipped
};
Result process();
@ -84,7 +88,8 @@ public:
fs::path const& _basepath,
fs::path const& _path,
string const& _ipcPath,
bool const _formatted
bool _formatted,
langutil::EVMVersion _evmVersion
);
static string editor;
@ -96,13 +101,14 @@ private:
Quit
};
Request handleResponse(bool const _exception);
Request handleResponse(bool _exception);
TestCase::TestCaseCreator m_testCaseCreator;
string const m_name;
fs::path const m_path;
string m_ipcPath;
bool const m_formatted = false;
langutil::EVMVersion const m_evmVersion;
unique_ptr<TestCase> m_test;
static bool m_exitRequested;
};
@ -119,8 +125,14 @@ TestTool::Result TestTool::process()
try
{
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath});
success = m_test->run(outputMessages, " ", m_formatted);
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath, m_evmVersion});
if (m_test->supportedForEVMVersion(m_evmVersion))
success = m_test->run(outputMessages, " ", m_formatted);
else
{
AnsiColorized(cout, m_formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
return Result::Skipped;
}
}
catch(boost::exception const& _e)
{
@ -158,7 +170,7 @@ TestTool::Result TestTool::process()
}
}
TestTool::Request TestTool::handleResponse(bool const _exception)
TestTool::Request TestTool::handleResponse(bool _exception)
{
if (_exception)
cout << "(e)dit/(s)kip/(q)uit? ";
@ -204,13 +216,15 @@ TestStats TestTool::processPath(
fs::path const& _basepath,
fs::path const& _path,
string const& _ipcPath,
bool const _formatted
bool _formatted,
langutil::EVMVersion _evmVersion
)
{
std::queue<fs::path> paths;
paths.push(_path);
int successCount = 0;
int testCount = 0;
int skippedCount = 0;
while (!paths.empty())
{
@ -235,7 +249,7 @@ TestStats TestTool::processPath(
else
{
++testCount;
TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted);
TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted, _evmVersion);
auto result = testTool.process();
switch(result)
@ -254,6 +268,7 @@ TestStats TestTool::processPath(
break;
case Request::Skip:
paths.pop();
++skippedCount;
break;
}
break;
@ -261,11 +276,15 @@ TestStats TestTool::processPath(
paths.pop();
++successCount;
break;
case Result::Skipped:
paths.pop();
++skippedCount;
break;
}
}
}
return { successCount, testCount };
return { successCount, testCount, skippedCount };
}
@ -298,7 +317,8 @@ boost::optional<TestStats> runTestSuite(
fs::path const& _subdirectory,
string const& _ipcPath,
TestCase::TestCaseCreator _testCaseCreator,
bool _formatted
bool _formatted,
langutil::EVMVersion _evmVersion
)
{
fs::path testPath = _basePath / _subdirectory;
@ -309,14 +329,21 @@ boost::optional<TestStats> runTestSuite(
return {};
}
TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted);
TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted, _evmVersion);
cout << endl << _name << " Test Summary: ";
AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
stats.successCount <<
"/" <<
stats.testCount;
cout << " tests successful." << endl << endl;
cout << " tests successful";
if (stats.skippedCount > 0)
{
cout << " (";
AnsiColorized(cout, _formatted, {BOLD, YELLOW}) << stats.skippedCount;
cout<< " tests skipped)";
}
cout << "." << endl << endl;
return stats;
}
@ -354,7 +381,7 @@ int main(int argc, char const *argv[])
if (ts.smt && options.disableSMT)
continue;
if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor))
if (auto stats = runTestSuite(ts.title, options.testPath / ts.path, ts.subpath, options.ipcPath.string(), ts.testCaseCreator, !options.noColor, options.evmVersion()))
global_stats += *stats;
else
return 1;
@ -363,7 +390,14 @@ int main(int argc, char const *argv[])
cout << endl << "Summary: ";
AnsiColorized(cout, !options.noColor, {BOLD, global_stats ? GREEN : RED}) <<
global_stats.successCount << "/" << global_stats.testCount;
cout << " tests successful." << endl;
cout << " tests successful";
if (global_stats.skippedCount > 0)
{
cout << " (";
AnsiColorized(cout, !options.noColor, {BOLD, YELLOW}) << global_stats.skippedCount;
cout << " tests skipped)";
}
cout << "." << endl;
return global_stats ? 0 : 1;
}