cerc-io/solidity
16
0
Fork 0
mirror of https://github.com/ethereum/solidity synced 2023-10-03 13:03:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor syntax test infrastructure to prepare introducing semantics tests.

Browse Source
This commit is contained in:
Daniel Kirchner 2018-06-08 14:17:50 +02:00
parent dea9646a1d
commit 14d0f8c2f1
7 changed files with 307 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
test/boostTest.cpp
View File

@ -38,7 +38,26 @@
#include <test/Options.h>
#include <test/libsolidity/SyntaxTest.h>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
using namespace boost::unit_test;
using namespace dev::solidity::test;
namespace fs = boost::filesystem;
using namespace std;
#if BOOST_VERSION < 105900
test_case *make_test_case(
function<void()> const& _fn,
string const& _name,
string const& /* _filename */,
size_t /* _line */
)
{
return make_test_case(_fn, _name);
}
#endif
namespace
{
@ -49,6 +68,56 @@ void removeTestSuite(std::string const& _name)
assert(id != INV_TEST_UNIT_ID);
master.remove(id);
}
int registerTests(
boost::unit_test::test_suite& _suite,
boost::filesystem::path const& _basepath,
boost::filesystem::path const& _path,
TestCase::TestCaseCreator _testCaseCreator
)
{
int numTestsAdded = 0;
fs::path fullpath = _basepath / _path;
if (fs::is_directory(fullpath))
{
test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
fs::directory_iterator(fullpath),
fs::directory_iterator()
))
if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _testCaseCreator);
_suite.add(sub_suite);
}
else
{
static vector<unique_ptr<string>> filenames;
filenames.emplace_back(new string(_path.string()));
_suite.add(make_test_case(
[fullpath, _testCaseCreator]
{
BOOST_REQUIRE_NO_THROW({
try
{
stringstream errorStream;
if (!_testCaseCreator(fullpath.string())->run(errorStream))
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
}
catch (boost::exception const& _e)
{
BOOST_ERROR("Exception during extracted test: " << boost::diagnostic_information(_e));
}
});
},
_path.stem().string(),
*filenames.back(),
0
));
numTestsAdded = 1;
}
return numTestsAdded;
}
}
test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
@ -56,10 +125,11 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
master_test_suite_t& master = framework::master_test_suite();
master.p_name.value = "SolidityTests";
dev::test::Options::get().validate();
solAssert(dev::solidity::test::SyntaxTest::registerTests(
solAssert(registerTests(
master,
dev::test::Options::get().testPath / "libsolidity",
"syntaxTests"
"syntaxTests",
SyntaxTest::create
) > 0, "no syntax tests found");
if (dev::test::Options::get().disableIPC)
{

152
test/libsolidity/SyntaxTest.cpp
View File

@ -33,26 +33,8 @@ using namespace std;
namespace fs = boost::filesystem;
using namespace boost::unit_test;
template<typename IteratorType>
void skipWhitespace(IteratorType& _it, IteratorType _end)
namespace
{
while (_it != _end && isspace(*_it))
++_it;
}
template<typename IteratorType>
void skipSlashes(IteratorType& _it, IteratorType _end)
{
while (_it != _end && *_it == '/')
++_it;
}
void expect(string::iterator& _it, string::iterator _end, string::value_type _c)
{
if (_it == _end || *_it != _c)
throw runtime_error(string("Invalid test expectation. Expected: \"") + _c + "\".");
++_it;
}
int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
{
@ -68,6 +50,8 @@ int parseUnsignedInteger(string::iterator &_it, string::iterator _end)
return result;
}
}
SyntaxTest::SyntaxTest(string const& _filename)
{
ifstream file(_filename);
@ -120,6 +104,55 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool const _fo
return true;
}
void SyntaxTest::printSource(ostream& _stream, string const& _linePrefix, bool const _formatted) const
{
if (_formatted)
{
if (m_source.empty())
return;
vector<char const*> sourceFormatting(m_source.length(), formatting::RESET);
for (auto const& error: m_errorList)
if (error.locationStart >= 0 && error.locationEnd >= 0)
{
assert(static_cast<size_t>(error.locationStart) <= m_source.length());
assert(static_cast<size_t>(error.locationEnd) <= m_source.length());
bool isWarning = error.type == "Warning";
for (int i = error.locationStart; i < error.locationEnd; i++)
if (isWarning)
{
if (sourceFormatting[i] == formatting::RESET)
sourceFormatting[i] = formatting::ORANGE_BACKGROUND;
}
else
sourceFormatting[i] = formatting::RED_BACKGROUND;
}
_stream << _linePrefix << sourceFormatting.front() << m_source.front();
for (size_t i = 1; i < m_source.length(); i++)
{
if (sourceFormatting[i] != sourceFormatting[i - 1])
_stream << sourceFormatting[i];
if (m_source[i] != '\n')
_stream << m_source[i];
else
{
_stream << formatting::RESET << endl;
if (i + 1 < m_source.length())
_stream << _linePrefix << sourceFormatting[i];
}
}
_stream << formatting::RESET;
}
else
{
stringstream stream(m_source);
string line;
while (getline(stream, line))
_stream << _linePrefix << line << endl;
}
}
void SyntaxTest::printErrorList(
ostream& _stream,
vector<SyntaxTestError> const& _errorList,
@ -159,19 +192,6 @@ string SyntaxTest::errorMessage(Exception const& _e)
return "NONE";
}
string SyntaxTest::parseSource(istream& _stream)
{
string source;
string line;
string const delimiter("// ----");
while (getline(_stream, line))
if (boost::algorithm::starts_with(line, delimiter))
break;
else
source += line + "\n";
return source;
}
vector<SyntaxTestError> SyntaxTest::parseExpectations(istream& _stream)
{
vector<SyntaxTestError> expectations;
@ -220,71 +240,3 @@ vector<SyntaxTestError> SyntaxTest::parseExpectations(istream& _stream)
}
return expectations;
}
#if BOOST_VERSION < 105900
test_case *make_test_case(
function<void()> const& _fn,
string const& _name,
string const& /* _filename */,
size_t /* _line */
)
{
return make_test_case(_fn, _name);
}
#endif
bool SyntaxTest::isTestFilename(boost::filesystem::path const& _filename)
{
return _filename.extension().string() == ".sol" &&
!boost::starts_with(_filename.string(), "~") &&
!boost::starts_with(_filename.string(), ".");
}
int SyntaxTest::registerTests(
boost::unit_test::test_suite& _suite,
boost::filesystem::path const& _basepath,
boost::filesystem::path const& _path
)
{
int numTestsAdded = 0;
fs::path fullpath = _basepath / _path;
if (fs::is_directory(fullpath))
{
test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
fs::directory_iterator(fullpath),
fs::directory_iterator()
))
if (fs::is_directory(entry.path()) || isTestFilename(entry.path().filename()))
numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
_suite.add(sub_suite);
}
else
{
static vector<unique_ptr<string>> filenames;
filenames.emplace_back(new string(_path.string()));
_suite.add(make_test_case(
[fullpath]
{
BOOST_REQUIRE_NO_THROW({
try
{
stringstream errorStream;
if (!SyntaxTest(fullpath.string()).run(errorStream))
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
}
catch (boost::exception const& _e)
{
BOOST_ERROR("Exception during syntax test: " << boost::diagnostic_information(_e));
}
});
},
_path.stem().string(),
*filenames.back(),
0
));
numTestsAdded = 1;
}
return numTestsAdded;
}

30
test/libsolidity/SyntaxTest.h
View File

@ -19,11 +19,9 @@
#include <test/libsolidity/AnalysisFramework.h>
#include <test/libsolidity/FormattedScope.h>
#include <test/libsolidity/TestCase.h>
#include <libsolidity/interface/Exceptions.h>
#include <boost/noncopyable.hpp>
#include <boost/test/unit_test.hpp>
#include <iosfwd>
#include <string>
#include <vector>
@ -52,17 +50,24 @@ struct SyntaxTestError
};
class SyntaxTest: AnalysisFramework
class SyntaxTest: AnalysisFramework, public TestCase
{
public:
static std::unique_ptr<TestCase> create(std::string const& _filename)
{ return std::unique_ptr<TestCase>(new SyntaxTest(_filename)); }
SyntaxTest(std::string const& _filename);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false);
virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
std::vector<SyntaxTestError> const& expectations() const { return m_expectations; }
std::string const& source() const { return m_source; }
std::vector<SyntaxTestError> const& errorList() const { return m_errorList; }
virtual void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
virtual void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override
{
if (!m_errorList.empty())
printErrorList(_stream, m_errorList, _linePrefix, false);
}
static std::string errorMessage(Exception const& _e);
private:
static void printErrorList(
std::ostream& _stream,
std::vector<SyntaxTestError> const& _errors,
@ -70,15 +75,6 @@ public:
bool const _formatted = false
);
static int registerTests(
boost::unit_test::test_suite& _suite,
boost::filesystem::path const& _basepath,
boost::filesystem::path const& _path
);
static bool isTestFilename(boost::filesystem::path const& _filename);
static std::string errorMessage(Exception const& _e);
private:
static std::string parseSource(std::istream& _stream);
static std::vector<SyntaxTestError> parseExpectations(std::istream& _stream);
std::string m_source;

55
test/libsolidity/TestCase.cpp Normal file
View File

@ -0,0 +1,55 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#include <test/libsolidity/TestCase.h>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <stdexcept>
using namespace dev;
using namespace solidity;
using namespace dev::solidity::test;
using namespace std;
bool TestCase::isTestFilename(boost::filesystem::path const& _filename)
{
return _filename.extension().string() == ".sol" &&
!boost::starts_with(_filename.string(), "~") &&
!boost::starts_with(_filename.string(), ".");
}
string TestCase::parseSource(istream& _stream)
{
string source;
string line;
string const delimiter("// ----");
while (getline(_stream, line))
if (boost::algorithm::starts_with(line, delimiter))
break;
else
source += line + "\n";
return source;
}
void TestCase::expect(string::iterator& _it, string::iterator _end, string::value_type _c)
{
if (_it == _end || *_it != _c)
throw runtime_error(string("Invalid test expectation. Expected: \"") + _c + "\".");
++_it;
}

80
test/libsolidity/TestCase.h Normal file
View File

@ -0,0 +1,80 @@
/*
This file is part of solidity.
solidity is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
solidity is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with solidity. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <boost/filesystem.hpp>
#include <iosfwd>
#include <memory>
#include <string>
namespace dev
{
namespace solidity
{
namespace test
{
/** Common superclass of SyntaxTest and SemanticsTest. */
class TestCase
{
public:
using TestCaseCreator = std::unique_ptr<TestCase>(*)(std::string const&);
virtual ~TestCase() {}
/// Runs the test case.
/// Outputs error messages to @arg _stream. Each line of output is prefixed with @arg _linePrefix.
/// Optionally, color-coding can be enabled (if @arg _formatted is set to true).
/// @returns true, if the test case succeeds, false otherwise
virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
/// Outputs the test contract to @arg _stream.
/// Each line of output is prefixed with @arg _linePrefix.
/// If @arg _formatted is true, color-coding may be used to indicate
/// error locations in the contract, if applicable.
virtual void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) const = 0;
/// Outputs test expectations to @arg _stream that match the actual results of the test.
/// Each line of output is prefixed with @arg _linePrefix.
virtual void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const = 0;
static bool isTestFilename(boost::filesystem::path const& _filename);
protected:
static std::string parseSource(std::istream& _file);
static void expect(std::string::iterator& _it, std::string::iterator _end, std::string::value_type _c);
template<typename IteratorType>
static void skipWhitespace(IteratorType& _it, IteratorType _end)
{
while (_it != _end && isspace(*_it))
++_it;
}
template<typename IteratorType>
static void skipSlashes(IteratorType& _it, IteratorType _end)
{
while (_it != _end && *_it == '/')
++_it;
}
};
}
}
}

4
test/tools/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_executable(solfuzzer fuzzer.cpp)
target_link_libraries(solfuzzer PRIVATE libsolc evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES} ${Boost_SYSTEM_LIBRARIES})
add_executable(isoltest isoltest.cpp ../Options.cpp ../libsolidity/SyntaxTest.cpp ../libsolidity/AnalysisFramework.cpp)
add_executable(isoltest isoltest.cpp ../Options.cpp ../libsolidity/TestCase.cpp ../libsolidity/SyntaxTest.cpp
../libsolidity/AnalysisFramework.cpp ../libsolidity/SolidityExecutionFramework.cpp ../ExecutionFramework.cpp
../RPCSession.cpp)
target_link_libraries(isoltest PRIVATE libsolc solidity evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})

110
test/tools/isoltest.cpp
View File

@ -37,18 +37,22 @@ using namespace std;
namespace po = boost::program_options;
namespace fs = boost::filesystem;
struct SyntaxTestStats
struct TestStats
{
int successCount;
int runCount;
operator bool() const { return successCount == runCount; }
};
class SyntaxTestTool
class TestTool
{
public:
SyntaxTestTool(string const& _name, fs::path const& _path, bool _formatted):
m_formatted(_formatted), m_name(_name), m_path(_path)
TestTool(
TestCase::TestCaseCreator _testCaseCreator,
string const& _name,
fs::path const& _path,
bool _formatted
): m_testCaseCreator(_testCaseCreator), m_formatted(_formatted), m_name(_name), m_path(_path)
{}
enum class Result
@ -60,7 +64,8 @@ public:
Result process();
static SyntaxTestStats processPath(
static TestStats processPath(
TestCase::TestCaseCreator _testCaseCreator,
fs::path const& _basepath,
fs::path const& _path,
bool const _formatted
@ -77,68 +82,16 @@ private:
Request handleResponse(bool const _exception);
void printContract() const;
TestCase::TestCaseCreator m_testCaseCreator;
bool const m_formatted;
string const m_name;
fs::path const m_path;
unique_ptr<SyntaxTest> m_test;
unique_ptr<TestCase> m_test;
};
string SyntaxTestTool::editor;
string TestTool::editor;
void SyntaxTestTool::printContract() const
{
if (m_formatted)
{
string const& source = m_test->source();
if (source.empty())
return;
std::vector<char const*> sourceFormatting(source.length(), formatting::RESET);
for (auto const& error: m_test->errorList())
if (error.locationStart >= 0 && error.locationEnd >= 0)
{
assert(static_cast<size_t>(error.locationStart) <= source.length());
assert(static_cast<size_t>(error.locationEnd) <= source.length());
bool isWarning = error.type == "Warning";
for (int i = error.locationStart; i < error.locationEnd; i++)
if (isWarning)
{
if (sourceFormatting[i] == formatting::RESET)
sourceFormatting[i] = formatting::ORANGE_BACKGROUND;
}
else
sourceFormatting[i] = formatting::RED_BACKGROUND;
}
cout << " " << sourceFormatting.front() << source.front();
for (size_t i = 1; i < source.length(); i++)
{
if (sourceFormatting[i] != sourceFormatting[i - 1])
cout << sourceFormatting[i];
if (source[i] != '\n')
cout << source[i];
else
{
cout << formatting::RESET << endl;
if (i + 1 < source.length())
cout << " " << sourceFormatting[i];
}
}
cout << formatting::RESET << endl;
}
else
{
stringstream stream(m_test->source());
string line;
while (getline(stream, line))
cout << " " << line << endl;
cout << endl;
}
}
SyntaxTestTool::Result SyntaxTestTool::process()
TestTool::Result TestTool::process()
{
bool success;
std::stringstream outputMessages;
@ -147,7 +100,7 @@ SyntaxTestTool::Result SyntaxTestTool::process()
try
{
m_test = unique_ptr<SyntaxTest>(new SyntaxTest(m_path.string()));
m_test = m_testCaseCreator(m_path.string());
success = m_test->run(outputMessages, " ", m_formatted);
}
catch(boost::exception const& _e)
@ -179,14 +132,14 @@ SyntaxTestTool::Result SyntaxTestTool::process()
FormattedScope(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl;
FormattedScope(cout, m_formatted, {BOLD, CYAN}) << " Contract:" << endl;
printContract();
m_test->printSource(cout, " ", m_formatted);
cout << outputMessages.str() << endl;
cout << endl << outputMessages.str() << endl;
return Result::Failure;
}
}
SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception)
TestTool::Request TestTool::handleResponse(bool const _exception)
{
if (_exception)
cout << "(e)dit/(s)kip/(q)uit? ";
@ -208,15 +161,14 @@ SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception)
{
cout << endl;
ofstream file(m_path.string(), ios::trunc);
file << m_test->source();
m_test->printSource(file);
file << "// ----" << endl;
if (!m_test->errorList().empty())
m_test->printErrorList(file, m_test->errorList(), "// ", false);
m_test->printUpdatedExpectations(file, "// ");
return Request::Rerun;
}
case 'e':
cout << endl << endl;
if (system((editor + " \"" + m_path.string() + "\"").c_str()))
if (system((TestTool::editor + " \"" + m_path.string() + "\"").c_str()))
cerr << "Error running editor command." << endl << endl;
return Request::Rerun;
case 'q':
@ -228,8 +180,8 @@ SyntaxTestTool::Request SyntaxTestTool::handleResponse(bool const _exception)
}
}
SyntaxTestStats SyntaxTestTool::processPath(
TestStats TestTool::processPath(
TestCase::TestCaseCreator _testCaseCreator,
fs::path const& _basepath,
fs::path const& _path,
bool const _formatted
@ -252,12 +204,12 @@ SyntaxTestStats SyntaxTestTool::processPath(
fs::directory_iterator(fullpath),
fs::directory_iterator()
))
if (fs::is_directory(entry.path()) || SyntaxTest::isTestFilename(entry.path().filename()))
if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
paths.push(currentPath / entry.path().filename());
}
else
{
SyntaxTestTool testTool(currentPath.string(), fullpath, _formatted);
TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _formatted);
++runCount;
auto result = testTool.process();
@ -293,9 +245,9 @@ SyntaxTestStats SyntaxTestTool::processPath(
int main(int argc, char *argv[])
{
if (getenv("EDITOR"))
SyntaxTestTool::editor = getenv("EDITOR");
TestTool::editor = getenv("EDITOR");
else if (fs::exists("/usr/bin/editor"))
SyntaxTestTool::editor = "/usr/bin/editor";
TestTool::editor = "/usr/bin/editor";
fs::path testPath;
bool formatted = true;
@ -311,7 +263,7 @@ Allowed options)",
("help", "Show this help screen.")
("testpath", po::value<fs::path>(&testPath), "path to test files")
("no-color", "don't use colors")
("editor", po::value<string>(&SyntaxTestTool::editor), "editor for opening contracts");
("editor", po::value<string>(&TestTool::editor), "editor for opening contracts");
po::variables_map arguments;
try
@ -331,7 +283,7 @@ Allowed options)",
po::notify(arguments);
}
catch (po::error const& _exception)
catch (std::exception const& _exception)
{
cerr << _exception.what() << endl;
return 1;
@ -362,7 +314,7 @@ Allowed options)",
if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath))
{
auto stats = SyntaxTestTool::processPath(testPath / "libsolidity", "syntaxTests", formatted);
auto stats = TestTool::processPath(SyntaxTest::create, testPath / "libsolidity", "syntaxTests", formatted);
cout << endl << "Summary: ";
FormattedScope(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
@ -373,7 +325,7 @@ Allowed options)",
}
else
{
cerr << "Test path not found. Use the --testpath argument." << endl;
cerr << "Syntax tests not found. Use the --testpath argument." << endl;
return 1;
}
}