Allow test cases to indicate fatal errors.

This commit is contained in:
Daniel Kirchner 2019-05-07 15:33:22 +02:00
parent 0a99519142
commit 76b88bdfd8
19 changed files with 75 additions and 59 deletions

View File

@ -56,6 +56,8 @@ public:
langutil::EVMVersion evmVersion;
};
enum class TestResult { Success, Failure, FatalError };
using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
virtual ~TestCase() = default;
@ -64,7 +66,7 @@ public:
/// Outputs error messages to @arg _stream. Each line of output is prefixed with @arg _linePrefix.
/// Optionally, color-coding can be enabled (if @arg _formatted is set to true).
/// @returns true, if the test case succeeds, false otherwise
virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
virtual TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) = 0;
/// Outputs the test contract to @arg _stream.
/// Each line of output is prefixed with @arg _linePrefix.

View File

@ -106,8 +106,17 @@ int registerTests(
stringstream errorStream;
auto testCase = _testCaseCreator(config);
if (testCase->validateSettings(dev::test::Options::get().evmVersion()))
if (!testCase->run(errorStream))
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
switch (testCase->run(errorStream))
{
case TestCase::TestResult::Success:
break;
case TestCase::TestResult::Failure:
BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
break;
case TestCase::TestResult::FatalError:
BOOST_ERROR("Fatal error during test.\n" + errorStream.str());
break;
}
}
catch (boost::exception const& _e)
{

View File

@ -18,6 +18,7 @@
#include <test/libsolidity/ASTJSONTest.h>
#include <test/Options.h>
#include <libdevcore/AnsiColorized.h>
#include <liblangutil/SourceReferenceFormatterHuman.h>
#include <libsolidity/ast/ASTJsonConverter.h>
#include <libsolidity/interface/CompilerStack.h>
#include <boost/algorithm/string.hpp>
@ -27,6 +28,7 @@
#include <memory>
#include <stdexcept>
using namespace langutil;
using namespace dev::solidity;
using namespace dev::solidity::test;
using namespace dev::formatting;
@ -88,7 +90,7 @@ ASTJSONTest::ASTJSONTest(string const& _filename)
}
}
bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
TestCase::TestResult ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
{
CompilerStack c;
@ -101,7 +103,15 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
}
c.setSources(sources);
c.setEVMVersion(dev::test::Options::get().evmVersion());
c.parseAndAnalyze();
if (c.parse())
c.analyze();
else
{
SourceReferenceFormatterHuman formatter(_stream, _formatted);
for (auto const& error: c.errors())
formatter.printErrorInformation(*error);
return TestResult::FatalError;
}
for (size_t i = 0; i < m_sources.size(); i++)
{
@ -169,7 +179,7 @@ bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _f
resultsMatch = false;
}
return resultsMatch;
return resultsMatch ? TestResult::Success : TestResult::Failure;
}
void ASTJSONTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

View File

@ -39,7 +39,7 @@ public:
{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
ASTJSONTest(std::string const& _filename);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
void printSource(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

View File

@ -158,7 +158,7 @@ void GasTest::printUpdatedExpectations(std::ostream& _stream, std::string const&
}
bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
TestCase::TestResult GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
string const versionPragma = "pragma solidity >=0.0;\n";
compiler().reset();
@ -174,10 +174,10 @@ bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
if (!compiler().parseAndAnalyze() || !compiler().compile())
{
SourceReferenceFormatterHuman formatter(cerr, _formatted);
SourceReferenceFormatterHuman formatter(_stream, _formatted);
for (auto const& error: compiler().errors())
formatter.printErrorInformation(*error);
BOOST_THROW_EXCEPTION(runtime_error("Test contract does not compile."));
return TestResult::FatalError;
}
Json::Value estimates = compiler().gasEstimates(compiler().lastContractName());
@ -226,7 +226,7 @@ bool GasTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
printUpdatedExpectations(_stream, _linePrefix + " ");
}
return success;
return success ? TestResult::Success : TestResult::Failure;
}
void GasTest::printSource(ostream& _stream, string const& _linePrefix, bool) const

View File

@ -40,7 +40,7 @@ public:
{ return std::make_unique<GasTest>(_config.filename); }
GasTest(std::string const& _filename);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

View File

@ -49,7 +49,7 @@ SMTCheckerTest::SMTCheckerTest(string const& _filename, langutil::EVMVersion _ev
BOOST_THROW_EXCEPTION(runtime_error("Invalid JSON file."));
}
bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
TestCase::TestResult SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
StandardCompiler compiler;
@ -121,7 +121,7 @@ bool SMTCheckerTest::run(ostream& _stream, string const& _linePrefix, bool _form
}
}
return printExpectationAndError(_stream, _linePrefix, _formatted);
return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
}
vector<string> SMTCheckerTest::hashesFromJson(Json::Value const& _jsonObj, string const& _auxInput, string const& _smtlib)

View File

@ -39,7 +39,7 @@ public:
}
SMTCheckerTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
private:
std::vector<std::string> hashesFromJson(Json::Value const& _jsonObj, std::string const& _auxInput, std::string const& _smtlib);

View File

@ -53,7 +53,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath, lang
parseExpectations(file);
}
bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract.");
@ -96,9 +96,9 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool _format
}
AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix << endl << _linePrefix
<< "Attention: Updates on the test will apply the detected format displayed." << endl;
return false;
return TestResult::Failure;
}
return true;
return TestResult::Success;
}
void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool) const

View File

@ -48,7 +48,7 @@ public:
explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath, langutil::EVMVersion _evmVersion);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override;

View File

@ -63,7 +63,7 @@ SyntaxTest::SyntaxTest(string const& _filename, langutil::EVMVersion _evmVersion
m_expectations = parseExpectations(file);
}
bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
TestCase::TestResult SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatted)
{
string const versionPragma = "pragma solidity >=0.0;\n";
compiler().reset();
@ -92,7 +92,7 @@ bool SyntaxTest::run(ostream& _stream, string const& _linePrefix, bool _formatte
});
}
return printExpectationAndError(_stream, _linePrefix, _formatted);
return printExpectationAndError(_stream, _linePrefix, _formatted) ? TestResult::Success : TestResult::Failure;
}
bool SyntaxTest::printExpectationAndError(ostream& _stream, string const& _linePrefix, bool _formatted)

View File

@ -57,7 +57,7 @@ public:
{ return std::make_unique<SyntaxTest>(_config.filename, _config.evmVersion); }
SyntaxTest(std::string const& _filename, langutil::EVMVersion _evmVersion);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
void printSource(std::ostream &_stream, std::string const &_linePrefix = "", bool _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override

View File

@ -62,7 +62,7 @@ ObjectCompilerTest::ObjectCompilerTest(string const& _filename)
m_expectation += line + "\n";
}
bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
TestCase::TestResult ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
{
AssemblyStack stack(
EVMVersion(),
@ -73,7 +73,7 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
{
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Error parsing source." << endl;
printErrors(_stream, stack.errors());
return false;
return TestResult::FatalError;
}
stack.optimize();
@ -98,9 +98,9 @@ bool ObjectCompilerTest::run(ostream& _stream, string const& _linePrefix, bool c
printIndented(_stream, m_expectation, nextIndentLevel);
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
printIndented(_stream, m_obtainedResult, nextIndentLevel);
return false;
return TestResult::Failure;
}
return true;
return TestResult::Success;
}
void ObjectCompilerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

View File

@ -47,7 +47,7 @@ public:
explicit ObjectCompilerTest(std::string const& _filename);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

View File

@ -67,10 +67,10 @@ YulInterpreterTest::YulInterpreterTest(string const& _filename)
m_expectation += line + "\n";
}
bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
TestCase::TestResult YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
{
if (!parse(_stream, _linePrefix, _formatted))
return false;
return TestResult::FatalError;
m_obtainedResult = interpret();
@ -82,9 +82,9 @@ bool YulInterpreterTest::run(ostream& _stream, string const& _linePrefix, bool c
printIndented(_stream, m_expectation, nextIndentLevel);
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
printIndented(_stream, m_obtainedResult, nextIndentLevel);
return false;
return TestResult::Failure;
}
return true;
return TestResult::Success;
}
void YulInterpreterTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

View File

@ -47,7 +47,7 @@ public:
explicit YulInterpreterTest(std::string const& _filename);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;

View File

@ -103,10 +103,10 @@ YulOptimizerTest::YulOptimizerTest(string const& _filename)
m_expectation += line + "\n";
}
bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
TestCase::TestResult YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
{
if (!parse(_stream, _linePrefix, _formatted))
return false;
return TestResult::FatalError;
if (m_optimizerStep == "disambiguator")
disambiguate();
@ -277,12 +277,11 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
else
{
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::RED}) << _linePrefix << "Invalid optimizer step: " << m_optimizerStep << endl;
return false;
return TestResult::FatalError;
}
m_obtainedResult = AsmPrinter{m_yul}(*m_ast) + "\n";
bool success = true;
if (m_optimizerStep != m_validatedSettings["step"])
{
string nextIndentLevel = _linePrefix + " ";
@ -294,7 +293,7 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
m_optimizerStep <<
"\"." <<
endl;
success = false;
return TestResult::FatalError;
}
if (m_expectation != m_obtainedResult)
{
@ -304,9 +303,9 @@ bool YulOptimizerTest::run(ostream& _stream, string const& _linePrefix, bool con
printIndented(_stream, m_expectation, nextIndentLevel);
AnsiColorized(_stream, _formatted, {formatting::BOLD, formatting::CYAN}) << _linePrefix << "Obtained result:" << endl;
printIndented(_stream, m_obtainedResult, nextIndentLevel);
success = false;
return TestResult::Failure;
}
return success;
return TestResult::Success;
}
void YulOptimizerTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const

View File

@ -48,7 +48,7 @@ public:
explicit YulOptimizerTest(std::string const& _filename);
bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
void printSource(std::ostream& _stream, std::string const &_linePrefix = "", bool const _formatted = false) const override;
void printUpdatedSettings(std::ostream &_stream, std::string const &_linePrefix = "", bool const _formatted = false) override;

View File

@ -147,7 +147,6 @@ bool TestTool::m_exitRequested = false;
TestTool::Result TestTool::process()
{
bool success;
bool formatted{!m_options.noColor};
std::stringstream outputMessages;
@ -159,7 +158,21 @@ TestTool::Result TestTool::process()
m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_options.ipcPath.string(), m_options.evmVersion()});
if (m_test->validateSettings(m_options.evmVersion()))
success = m_test->run(outputMessages, " ", formatted);
switch (TestCase::TestResult result = m_test->run(outputMessages, " ", formatted))
{
case TestCase::TestResult::Success:
AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
return Result::Success;
default:
AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
m_test->printSource(cout, " ", formatted);
m_test->printUpdatedSettings(cout, " ", formatted);
cout << endl << outputMessages.str() << endl;
return result == TestCase::TestResult::FatalError ? Result::Exception : Result::Failure;
}
else
{
AnsiColorized(cout, formatted, {BOLD, YELLOW}) << "NOT RUN" << endl;
@ -187,23 +200,6 @@ TestTool::Result TestTool::process()
"Unknown exception during test." << endl;
return Result::Exception;
}
if (success)
{
AnsiColorized(cout, formatted, {BOLD, GREEN}) << "OK" << endl;
return Result::Success;
}
else
{
AnsiColorized(cout, formatted, {BOLD, RED}) << "FAIL" << endl;
AnsiColorized(cout, formatted, {BOLD, CYAN}) << " Contract:" << endl;
m_test->printSource(cout, " ", formatted);
m_test->printUpdatedSettings(cout, " ", formatted);
cout << endl << outputMessages.str() << endl;
return Result::Failure;
}
}
TestTool::Request TestTool::handleResponse(bool _exception)