mirror of
https://github.com/ethereum/solidity
synced 2023-10-03 13:03:40 +00:00
Allow test cases to indicate fatal errors.
This commit is contained in:
parent
0a99519142
commit
76b88bdfd8
test
TestCase.hboostTest.cpp
libsolidity
ASTJSONTest.cppASTJSONTest.hGasTest.cppGasTest.hSMTCheckerJSONTest.cppSMTCheckerJSONTest.hSemanticTest.cppSemanticTest.hSyntaxTest.cppSyntaxTest.h
libyul
ObjectCompilerTest.cppObjectCompilerTest.hYulInterpreterTest.cppYulInterpreterTest.hYulOptimizerTest.cppYulOptimizerTest.h
tools
@ -56,6 +56,8 @@ public:
|
||||
langutil::EVMVersion evmVersion;
|
||||
};
|
||||
|
||||
enum class TestResult { Success, Failure, FatalError };
|
||||
|
||||
using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
|
||||
|
||||
virtual ~TestCase() = default;
|
||||
@ -64,7 +66,7 @@ public:
|
||||
/// Outputs error messages to @arg _stream. Each line of output is prefixed with @arg _linePrefix.
|
||||
/// Optionally, color-coding can be enabled (if @arg _formatted is set to true).
|
||||
/// @returns true, if the test case succeeds, false otherwise
|
||||
virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
|
||||
virtual TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
|
||||
|
||||
/// Outputs the test contract to @arg _stream.
|
||||
/// Each line of output is prefixed with @arg _linePrefix.
|
||||
|
@ -106,8 +106,17 @@ int registerTests(
|
||||
stringstream errorStream;
|
||||
auto testCase = _testCaseCreator(config);
|
||||
if (testCase->validateSettings(dev::test::Options::get().evmVersion()))
|
||||
if (!testCase->run(errorStream))
|
||||
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
|
||||
switch (testCase->run(errorStream))
|
||||
{
|
||||
case TestCase::TestResult::Success:
|
||||
break;
|
||||
case TestCase::TestResult::Failure:
|
||||
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
|
||||
break;
|
||||
case TestCase::TestResult::FatalError:
|
||||
BOOST_ERROR("Fatal error during test.\n" + errorStream.str());
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (boost::exception const& _e)
|
||||
{
|
||||
|
@ -18,6 +18,7 @@
|
||||
#include <test/libsolidity/ASTJSONTest.h>
|
||||
#include <test/Options.h>
|
||||
#include <libdevcore/AnsiColorized.h>
|
||||
#include <liblangutil/SourceReferenceFormatterHuman.h>
|
||||
#include <libsolidity/ast/ASTJsonConverter.h>
|
||||
#include <libsolidity/interface/CompilerStack.h>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
@ -27,6 +28,7 @@
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
|
||||
using namespace langutil;
|
||||
using namespace dev::solidity;
|
||||
using namespace dev::solidity::test;
|
||||
using namespace dev::formatting;
|
||||
@ -88,7 +90,7 @@ ASTJSONTest::ASTJSONTest(string const& _filename)
|
||||
}
|
||||
}
|
||||
|
||||
bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
TestCase::TestResult ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
{
|
||||
CompilerStack c;
|
||||
|
||||
@ -101,7 +103,15 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
|
||||
}
|
||||
c.setSources(sources);
|
||||
c.setEVMVersion(dev::test::Options::get().evmVersion());
|
||||
c.parseAndAnalyze();
|
||||
if (c.parse())
|
||||
c.analyze();
|
||||
else
|
||||
{
|
||||
SourceReferenceFormatterHuman formatter(_stream, _formatted);
|
||||
for (auto const& error: c.errors())
|
||||
formatter.printErrorInformation(*error);
|
||||
return TestResult::FatalError;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < m_sources.size(); i++)
|
||||
{
|
||||
@ -169,7 +179,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
|
||||
resultsMatch = false;
|
||||
}
|
||||
|
||||
return resultsMatch;
|
||||
return resultsMatch ? TestResult::Success : TestResult::Failure;
|
||||
}
|
||||
|
||||
void ASTJSONTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
|
||||
|
@ -39,7 +39,7 @@ public:
|
||||
{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
|
||||
ASTJSONTest(std::string const& _filename);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
|
||||
void printSource(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
|
||||
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
|
||||
|
@ -158,7 +158,7 @@ void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const&
|
||||
}
|
||||
|
||||
|
||||
bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
TestCase::TestResult GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
{
|
||||
string const versionPragma = "pragma solidity >=0.0;\n";
|
||||
compiler().reset();
|
||||
@ -174,10 +174,10 @@ bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
|
||||
if (!compiler().parseAndAnalyze() || !compiler().compile())
|
||||
{
|
||||
SourceReferenceFormatterHuman formatter(cerr, _formatted);
|
||||
SourceReferenceFormatterHuman formatter(_stream, _formatted);
|
||||
for (auto const& error: compiler().errors())
|
||||
formatter.printErrorInformation(*error);
|
||||
BOOST_THROW_EXCEPTION(runtime_error("Test contract does not compile."));
|
||||
return TestResult::FatalError;
|
||||
}
|
||||
|
||||
Json::Value estimates = compiler().gasEstimates(compiler().lastContractName());
|
||||
@ -226,7 +226,7 @@ bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
printUpdatedExpectations(_stream, _linePrefix + " ");
|
||||
}
|
||||
|
||||
return success;
|
||||
return success ? TestResult::Success : TestResult::Failure;
|
||||
}
|
||||
|
||||
void GasTest::printSource(ostream& _stream, string const& _linePrefix, bool) const
|
||||
|
@ -40,7 +40,7 @@ public:
|
||||
{ return std::make_unique<GasTest>(_config.filename); }
|
||||
GasTest(std::string const& _filename);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
|
||||
void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
|
||||
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
|
||||
|
@ -49,7 +49,7 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev
|
||||
BOOST_THROW_EXCEPTION(runtime_error("Invalid JSON file."));
|
||||
}
|
||||
|
||||
bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
{
|
||||
StandardCompiler compiler;
|
||||
|
||||
@ -121,7 +121,7 @@ bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _form
|
||||
}
|
||||
}
|
||||
|
||||
return printExpectationAndError(_stream, _linePrefix, _formatted);
|
||||
return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
|
||||
}
|
||||
|
||||
vector<string> SMTCheckerTest::hashesFromJson(Json::Value const& _jsonObj, string const& _auxInput, string const& _smtlib)
|
||||
|
@ -39,7 +39,7 @@ public:
|
||||
}
|
||||
SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
|
||||
private:
|
||||
std::vector<std::string> hashesFromJson(Json::Value const& _jsonObj, std::string const& _auxInput, std::string const& _smtlib);
|
||||
|
@ -53,7 +53,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, lang
|
||||
parseExpectations(file);
|
||||
}
|
||||
|
||||
bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
{
|
||||
soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
|
||||
|
||||
@ -96,9 +96,9 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _format
|
||||
}
|
||||
AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix << endl << _linePrefix
|
||||
<< "Attention: Updates on the test will apply the detected format displayed." << endl;
|
||||
return false;
|
||||
return TestResult::Failure;
|
||||
}
|
||||
return true;
|
||||
return TestResult::Success;
|
||||
}
|
||||
|
||||
void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const
|
||||
|
@ -48,7 +48,7 @@ public:
|
||||
|
||||
explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
|
||||
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;
|
||||
|
||||
|
@ -63,7 +63,7 @@ SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion
|
||||
m_expectations = parseExpectations(file);
|
||||
}
|
||||
|
||||
bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
TestCase::TestResult SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
{
|
||||
string const versionPragma = "pragma solidity >=0.0;\n";
|
||||
compiler().reset();
|
||||
@ -92,7 +92,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatte
|
||||
});
|
||||
}
|
||||
|
||||
return printExpectationAndError(_stream, _linePrefix, _formatted);
|
||||
return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
|
||||
}
|
||||
|
||||
bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool _formatted)
|
||||
|
@ -57,7 +57,7 @@ public:
|
||||
{ return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion); }
|
||||
SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
|
||||
|
||||
void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
|
||||
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override
|
||||
|
@ -62,7 +62,7 @@ ObjectCompilerTest::ObjectCompilerTest(string const& _filename)
|
||||
m_expectation += line + "\n";
|
||||
}
|
||||
|
||||
bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
{
|
||||
AssemblyStack stack(
|
||||
EVMVersion(),
|
||||
@ -73,7 +73,7 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
|
||||
{
|
||||
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Error parsing source." << endl;
|
||||
printErrors(_stream, stack.errors());
|
||||
return false;
|
||||
return TestResult::FatalError;
|
||||
}
|
||||
stack.optimize();
|
||||
|
||||
@ -98,9 +98,9 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
|
||||
printIndented(_stream, m_expectation, nextIndentLevel);
|
||||
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
|
||||
printIndented(_stream, m_obtainedResult, nextIndentLevel);
|
||||
return false;
|
||||
return TestResult::Failure;
|
||||
}
|
||||
return true;
|
||||
return TestResult::Success;
|
||||
}
|
||||
|
||||
void ObjectCompilerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
|
||||
|
@ -47,7 +47,7 @@ public:
|
||||
|
||||
explicit ObjectCompilerTest(std::string const& _filename);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
|
||||
void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
|
||||
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
|
||||
|
@ -67,10 +67,10 @@ YulInterpreterTest::YulInterpreterTest(string const& _filename)
|
||||
m_expectation += line + "\n";
|
||||
}
|
||||
|
||||
bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
TestCase::TestResult YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
{
|
||||
if (!parse(_stream, _linePrefix, _formatted))
|
||||
return false;
|
||||
return TestResult::FatalError;
|
||||
|
||||
m_obtainedResult = interpret();
|
||||
|
||||
@ -82,9 +82,9 @@ bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool c
|
||||
printIndented(_stream, m_expectation, nextIndentLevel);
|
||||
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
|
||||
printIndented(_stream, m_obtainedResult, nextIndentLevel);
|
||||
return false;
|
||||
return TestResult::Failure;
|
||||
}
|
||||
return true;
|
||||
return TestResult::Success;
|
||||
}
|
||||
|
||||
void YulInterpreterTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
|
||||
|
@ -47,7 +47,7 @@ public:
|
||||
|
||||
explicit YulInterpreterTest(std::string const& _filename);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
|
||||
void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
|
||||
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
|
||||
|
@ -103,10 +103,10 @@ YulOptimizerTest::YulOptimizerTest(string const& _filename)
|
||||
m_expectation += line + "\n";
|
||||
}
|
||||
|
||||
bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
|
||||
{
|
||||
if (!parse(_stream, _linePrefix, _formatted))
|
||||
return false;
|
||||
return TestResult::FatalError;
|
||||
|
||||
if (m_optimizerStep == "disambiguator")
|
||||
disambiguate();
|
||||
@ -277,12 +277,11 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
|
||||
else
|
||||
{
|
||||
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Invalid optimizer step: " << m_optimizerStep << endl;
|
||||
return false;
|
||||
return TestResult::FatalError;
|
||||
}
|
||||
|
||||
m_obtainedResult = AsmPrinter{m_yul}(*m_ast) + "\n";
|
||||
|
||||
bool success = true;
|
||||
if (m_optimizerStep != m_validatedSettings["step"])
|
||||
{
|
||||
string nextIndentLevel = _linePrefix + " ";
|
||||
@ -294,7 +293,7 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
|
||||
m_optimizerStep <<
|
||||
"\"." <<
|
||||
endl;
|
||||
success = false;
|
||||
return TestResult::FatalError;
|
||||
}
|
||||
if (m_expectation != m_obtainedResult)
|
||||
{
|
||||
@ -304,9 +303,9 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
|
||||
printIndented(_stream, m_expectation, nextIndentLevel);
|
||||
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
|
||||
printIndented(_stream, m_obtainedResult, nextIndentLevel);
|
||||
success = false;
|
||||
return TestResult::Failure;
|
||||
}
|
||||
return success;
|
||||
return TestResult::Success;
|
||||
}
|
||||
|
||||
void YulOptimizerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
|
||||
|
@ -48,7 +48,7 @@ public:
|
||||
|
||||
explicit YulOptimizerTest(std::string const& _filename);
|
||||
|
||||
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
|
||||
|
||||
void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
|
||||
void printUpdatedSettings(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) override;
|
||||
|
@ -147,7 +147,6 @@ bool TestTool::m_exitRequested = false;
|
||||
|
||||
TestTool::Result TestTool::process()
|
||||
{
|
||||
bool success;
|
||||
bool formatted{!m_options.noColor};
|
||||
std::stringstream outputMessages;
|
||||
|
||||
@ -159,7 +158,21 @@ TestTool::Result TestTool::process()
|
||||
|
||||
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
|
||||
if (m_test->validateSettings(m_options.evmVersion()))
|
||||
success = m_test->run(outputMessages, " ", formatted);
|
||||
switch (TestCase::TestResult result = m_test->run(outputMessages, " ", formatted))
|
||||
{
|
||||
case TestCase::TestResult::Success:
|
||||
AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
|
||||
return Result::Success;
|
||||
default:
|
||||
AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
|
||||
|
||||
AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
|
||||
m_test->printSource(cout, " ", formatted);
|
||||
m_test->printUpdatedSettings(cout, " ", formatted);
|
||||
|
||||
cout << endl << outputMessages.str() << endl;
|
||||
return result == TestCase::TestResult::FatalError ? Result::Exception : Result::Failure;
|
||||
}
|
||||
else
|
||||
{
|
||||
AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
|
||||
@ -187,23 +200,6 @@ TestTool::Result TestTool::process()
|
||||
"Unknown exception during test." << endl;
|
||||
return Result::Exception;
|
||||
}
|
||||
|
||||
if (success)
|
||||
{
|
||||
AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
|
||||
return Result::Success;
|
||||
}
|
||||
else
|
||||
{
|
||||
AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
|
||||
|
||||
AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
|
||||
m_test->printSource(cout, " ", formatted);
|
||||
m_test->printUpdatedSettings(cout, " ", formatted);
|
||||
|
||||
cout << endl << outputMessages.str() << endl;
|
||||
return Result::Failure;
|
||||
}
|
||||
}
|
||||
|
||||
TestTool::Request TestTool::handleResponse(bool _exception)
|
||||
|
Loading…
Reference in New Issue
Block a user