[isoltest] Adding gas used as semantic tests expectation.

This commit is contained in:
Djordje Mijovic 2021-03-09 21:26:36 +01:00
parent ad5d34df74
commit 6d51dfb617
7 changed files with 185 additions and 109 deletions

View File

@ -133,6 +133,7 @@ TestCase::TestResult SemanticTest::runTest(
)
{
bool success = true;
m_gasCostFailure = false;
if (_compileViaYul && _compileToEwasm)
selectVM(evmc_capabilities::EVMC_CAPABILITY_EWASM);
@ -203,6 +204,8 @@ TestCase::TestResult SemanticTest::runTest(
{
if (m_transactionSuccessful == test.call().expectations.failure)
success = false;
if (success && !checkGasCostExpectation(test, _compileViaYul))
m_gasCostFailure = true;
test.setFailure(!m_transactionSuccessful);
test.setRawBytes(bytes());
@ -239,6 +242,9 @@ TestCase::TestResult SemanticTest::runTest(
}
bool outputMismatch = (output != test.call().expectations.rawBytes());
if (!outputMismatch && !checkGasCostExpectation(test, _compileViaYul))
m_gasCostFailure = true;
// Pre byzantium, it was not possible to return failure data, so we disregard
// output mismatch for those EVM versions.
if (test.call().expectations.failure && !m_transactionSuccessful && !m_evmVersion.supportsReturndata())
@ -297,9 +303,33 @@ TestCase::TestResult SemanticTest::runTest(
return TestResult::Failure;
}
if (m_gasCostFailure)
{
AnsiColorized(_stream, _formatted, {BOLD, CYAN})
<< _linePrefix << "Gas results missing or wrong, obtained result:" << endl;
for (auto const& test: m_tests)
{
ErrorReporter errorReporter;
_stream << test.format(errorReporter, _linePrefix, false, _formatted) << endl;
_stream << errorReporter.format(_linePrefix, _formatted);
}
return TestResult::Failure;
}
return TestResult::Success;
}
bool SemanticTest::checkGasCostExpectation(TestFunctionCall& io_test, bool _compileViaYul) const
{
string setting =
(_compileViaYul ? "ir"s : "legacy"s) +
(m_optimiserSettings == OptimiserSettings::full() ? "Optimized" : "");
io_test.setGasCost(setting, m_gasUsed);
return
io_test.call().expectations.gasUsed.count(setting) > 0 &&
m_gasUsed == io_test.call().expectations.gasUsed.at(setting);
}
void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool _formatted) const
{
if (m_sources.sources.empty())
@ -347,7 +377,7 @@ void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool
void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
{
for (TestFunctionCall const& test: m_tests)
_stream << test.format("", true, false) << endl;
_stream << test.format("", /* _renderResult = */ !m_gasCostFailure, /* _highlight = */ false) << endl;
}
void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePrefix)

View File

@ -61,6 +61,7 @@ public:
private:
TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm);
bool checkGasCostExpectation(TestFunctionCall& io_test, bool _compileViaYul) const;
SourceMap m_sources;
std::size_t m_lineOffset;
std::vector<TestFunctionCall> m_tests;
@ -72,6 +73,8 @@ private:
bool m_allowNonExistingFunctions = false;
bool m_compileViaYulCanBeSet = false;
std::map<std::string, Builtin> m_builtins{};
bool m_gasCostFailure = false;
};
}

View File

@ -25,41 +25,42 @@ namespace solidity::frontend::test
/**
* All soltest tokens.
*/
#define SOLT_TOKEN_LIST(T, K) \
T(Unknown, "unknown", 0) \
T(Invalid, "invalid", 0) \
T(EOS, "EOS", 0) \
T(Whitespace, "_", 0) \
/* punctuations */ \
T(LParen, "(", 0) \
T(RParen, ")", 0) \
T(LBrack, "[", 0) \
T(RBrack, "]", 0) \
T(LBrace, "{", 0) \
T(RBrace, "}", 0) \
T(Sub, "-", 0) \
T(Colon, ":", 0) \
T(Comma, ",", 0) \
T(Period, ".", 0) \
T(Arrow, "->", 0) \
T(Newline, "//", 0) \
/* Literals & identifier */ \
T(Comment, "#", 0) \
T(Number, "number", 0) \
T(HexNumber, "hex_number", 0) \
T(String, "string", 0) \
T(Identifier, "identifier", 0) \
/* type keywords */ \
K(Ether, "ether", 0) \
K(Wei, "wei", 0) \
K(Hex, "hex", 0) \
K(Boolean, "boolean", 0) \
/* special keywords */ \
K(Left, "left", 0) \
K(Library, "library", 0) \
K(Right, "right", 0) \
K(Failure, "FAILURE", 0) \
K(Storage, "storage", 0) \
#define SOLT_TOKEN_LIST(T, K) \
T(Unknown, "unknown", 0) \
T(Invalid, "invalid", 0) \
T(EOS, "EOS", 0) \
T(Whitespace, "_", 0) \
/* punctuations */ \
T(LParen, "(", 0) \
T(RParen, ")", 0) \
T(LBrack, "[", 0) \
T(RBrack, "]", 0) \
T(LBrace, "{", 0) \
T(RBrace, "}", 0) \
T(Sub, "-", 0) \
T(Colon, ":", 0) \
T(Comma, ",", 0) \
T(Period, ".", 0) \
T(Arrow, "->", 0) \
T(Newline, "//", 0) \
/* Literals & identifier */ \
T(Comment, "#", 0) \
T(Number, "number", 0) \
T(HexNumber, "hex_number", 0) \
T(String, "string", 0) \
T(Identifier, "identifier", 0) \
/* type keywords */ \
K(Ether, "ether", 0) \
K(Wei, "wei", 0) \
K(Hex, "hex", 0) \
K(Boolean, "boolean", 0) \
/* special keywords */ \
K(Left, "left", 0) \
K(Library, "library", 0) \
K(Right, "right", 0) \
K(Failure, "FAILURE", 0) \
K(Storage, "storage", 0) \
K(Gas, "gas", 0) \
namespace soltest
{
@ -207,6 +208,9 @@ struct FunctionCallExpectations
raw += param.rawBytes;
return raw;
}
/// Gas used by function call
/// Should have values for Yul, YulOptimized, Legacy and LegacyOptimized
std::map<std::string, u256> gasUsed;
};
/**

View File

@ -62,8 +62,6 @@ vector<solidity::frontend::test::FunctionCall> TestFileParser::parseFunctionCall
{
if (!accept(Token::Whitespace))
{
FunctionCall call;
/// If this is not the first call in the test,
/// the last call to parseParameter could have eaten the
/// new line already. This could only be fixed with a one
@ -77,78 +75,100 @@ vector<solidity::frontend::test::FunctionCall> TestFileParser::parseFunctionCall
try
{
if (accept(Token::Library, true))
if (accept(Token::Gas, true))
{
expect(Token::Colon);
call.signature = m_scanner.currentLiteral();
expect(Token::Identifier);
call.kind = FunctionCall::Kind::Library;
call.expectations.failure = false;
}
else if (accept(Token::Storage, true))
{
expect(Token::Colon);
call.expectations.failure = false;
call.expectations.result.push_back(Parameter());
// empty / non-empty is encoded as false / true
if (m_scanner.currentLiteral() == "empty")
call.expectations.result.back().rawBytes = bytes(1, uint8_t(false));
else if (m_scanner.currentLiteral() == "nonempty")
call.expectations.result.back().rawBytes = bytes(1, uint8_t(true));
if (calls.empty())
BOOST_THROW_EXCEPTION(TestParserError("Expected function call before gas usage filter."));
string runType = m_scanner.currentLiteral();
if (set<string>{"ir", "irOptimized", "legacy", "legacyOptimized"}.count(runType) > 0)
{
m_scanner.scanNextToken();
expect(Token::Colon);
if (calls.back().expectations.gasUsed.count(runType) > 0)
throw TestParserError("Gas usage expectation set multiple times.");
calls.back().expectations.gasUsed[runType] = u256(parseDecimalNumber());
}
else
BOOST_THROW_EXCEPTION(TestParserError("Expected \"empty\" or \"nonempty\"."));
call.kind = FunctionCall::Kind::Storage;
m_scanner.scanNextToken();
BOOST_THROW_EXCEPTION(TestParserError(
"Expected \"ir\", \"irOptimized\", \"legacy\", or \"legacyOptimized\"."
));
}
else
{
bool lowLevelCall = false;
tie(call.signature, lowLevelCall) = parseFunctionSignature();
if (lowLevelCall)
call.kind = FunctionCall::Kind::LowLevel;
else if (isBuiltinFunction(call.signature))
call.kind = FunctionCall::Kind::Builtin;
FunctionCall call;
if (accept(Token::Comma, true))
call.value = parseFunctionCallValue();
if (accept(Token::Colon, true))
call.arguments = parseFunctionCallArguments();
if (accept(Token::Newline, true))
if (accept(Token::Library, true))
{
call.displayMode = FunctionCall::DisplayMode::MultiLine;
m_lineNumber++;
expect(Token::Colon);
call.signature = m_scanner.currentLiteral();
expect(Token::Identifier);
call.kind = FunctionCall::Kind::Library;
call.expectations.failure = false;
}
call.arguments.comment = parseComment();
if (accept(Token::Newline, true))
else if (accept(Token::Storage, true))
{
call.displayMode = FunctionCall::DisplayMode::MultiLine;
m_lineNumber++;
}
if (accept(Token::Arrow, true))
{
call.omitsArrow = false;
call.expectations = parseFunctionCallExpectations();
if (accept(Token::Newline, true))
m_lineNumber++;
expect(Token::Colon);
call.expectations.failure = false;
call.expectations.result.push_back(Parameter());
// empty / non-empty is encoded as false / true
if (m_scanner.currentLiteral() == "empty")
call.expectations.result.back().rawBytes = bytes(1, uint8_t(false));
else if (m_scanner.currentLiteral() == "nonempty")
call.expectations.result.back().rawBytes = bytes(1, uint8_t(true));
else
BOOST_THROW_EXCEPTION(TestParserError("Expected \"empty\" or \"nonempty\"."));
call.kind = FunctionCall::Kind::Storage;
m_scanner.scanNextToken();
}
else
{
call.expectations.failure = false;
call.displayMode = FunctionCall::DisplayMode::SingleLine;
bool lowLevelCall = false;
tie(call.signature, lowLevelCall) = parseFunctionSignature();
if (lowLevelCall)
call.kind = FunctionCall::Kind::LowLevel;
if (accept(Token::Comma, true))
call.value = parseFunctionCallValue();
if (accept(Token::Colon, true))
call.arguments = parseFunctionCallArguments();
if (accept(Token::Newline, true))
{
call.displayMode = FunctionCall::DisplayMode::MultiLine;
m_lineNumber++;
}
call.arguments.comment = parseComment();
if (accept(Token::Newline, true))
{
call.displayMode = FunctionCall::DisplayMode::MultiLine;
m_lineNumber++;
}
if (accept(Token::Arrow, true))
{
call.omitsArrow = false;
call.expectations = parseFunctionCallExpectations();
if (accept(Token::Newline, true))
m_lineNumber++;
}
else
{
call.expectations.failure = false;
call.displayMode = FunctionCall::DisplayMode::SingleLine;
}
call.expectations.comment = parseComment();
if (call.signature == "constructor()")
call.kind = FunctionCall::Kind::Constructor;
}
call.expectations.comment = parseComment();
if (call.signature == "constructor()")
call.kind = FunctionCall::Kind::Constructor;
calls.emplace_back(std::move(call));
}
calls.emplace_back(std::move(call));
}
catch (TestParserError const& _e)
{
@ -506,6 +526,7 @@ void TestFileParser::Scanner::scanNextToken()
if (_literal == "hex") return {Token::Hex, ""};
if (_literal == "FAILURE") return {Token::Failure, ""};
if (_literal == "storage") return {Token::Storage, ""};
if (_literal == "gas") return {Token::Gas, ""};
return {Token::Identifier, _literal};
};

View File

@ -93,7 +93,6 @@ string TestFunctionCall::format(
if (!m_call.arguments.parameters.at(0).format.newline)
stream << ws;
stream << output;
}
/// Formats comments on the function parameters and the arrow taking
@ -204,6 +203,8 @@ string TestFunctionCall::format(
stream << comment << m_call.expectations.comment << comment;
}
}
stream << formatGasExpectations(_linePrefix);
};
formatOutput(m_call.displayMode == FunctionCall::DisplayMode::SingleLine);
@ -319,6 +320,17 @@ string TestFunctionCall::formatRawParameters(
return os.str();
}
string TestFunctionCall::formatGasExpectations(string const& _linePrefix) const
{
stringstream os;
for (auto const& [runType, gasUsed]: m_call.expectations.gasUsed)
if (runType != get<0>(m_gasCost))
os << endl << _linePrefix << "// gas " << runType << ": " << gasUsed.str();
if (!get<0>(m_gasCost).empty())
os << endl << _linePrefix << "// gas " << get<0>(m_gasCost) << ": " << get<1>(m_gasCost).str();
return os.str();
}
void TestFunctionCall::reset()
{
m_rawBytes = bytes{};

View File

@ -79,6 +79,7 @@ public:
void calledNonExistingFunction() { m_calledNonExistingFunction = true; }
void setFailure(const bool _failure) { m_failure = _failure; }
void setRawBytes(const bytes _rawBytes) { m_rawBytes = _rawBytes; }
void setGasCost(std::string _runType, u256 _gasCost) { m_gasCost = {std::move(_runType), std::move(_gasCost)}; }
void setContractABI(Json::Value _contractABI) { m_contractABI = std::move(_contractABI); }
private:
@ -116,6 +117,9 @@ private:
std::string const& _linePrefix = ""
) const;
/// Formats gas usage expectations one per line
std::string formatGasExpectations(std::string const& _linePrefix) const;
/// Compares raw expectations (which are converted to a byte representation before),
/// and also the expected transaction status of the function call to the actual test results.
bool matchesExpectation() const;
@ -124,6 +128,8 @@ private:
FunctionCall m_call;
/// Result of the actual call been made.
bytes m_rawBytes = bytes{};
/// Actual gas cost for the type of the run
std::tuple<std::string, u256> m_gasCost;
/// Transaction status of the actual call. False in case of a REVERT or any other failure.
bool m_failure = true;
/// JSON object which holds the contract ABI and that is used to set the output formatting

View File

@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline)
bytes expectedBytes = toBigEndian(u256{1});
ABIType abiType{ABIType::UnsignedDec, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(uint8)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -59,7 +59,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline_signed_encoding)
bytes expectedBytes = toBigEndian(u256{1});
ABIType abiType{ABIType::UnsignedDec, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(uint8)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_multiline)
bytes expectedBytes = toBigEndian(u256{1});
ABIType abiType{ABIType::UnsignedDec, ABIType::AlignRight, 32};
Parameter result{expectedBytes, "1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{result}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{result}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{}, string{}};
FunctionCall call{"f(uint8)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(format_multiple_unsigned_singleline)
bytes expectedBytes = toBigEndian(u256{1});
ABIType abiType{ABIType::UnsignedDec, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param, param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param, param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param, param}, string{}};
FunctionCall call{"f(uint8, uint8)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -108,7 +108,7 @@ BOOST_AUTO_TEST_CASE(format_signed_singleline)
bytes expectedBytes = toBigEndian(u256{-1});
ABIType abiType{ABIType::UnsignedDec, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "-1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(int8)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -128,7 +128,7 @@ BOOST_AUTO_TEST_CASE(format_hex_singleline)
bytes expectedBytes = result + bytes(32 - result.size(), 0);
ABIType abiType{ABIType::Hex, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "0x31", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(bytes32)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -150,7 +150,7 @@ BOOST_AUTO_TEST_CASE(format_hex_string_singleline)
bytes expectedBytes = fromHex("4200ef");
ABIType abiType{ABIType::HexString, ABIType::AlignLeft, 3};
Parameter param{expectedBytes, "hex\"4200ef\"", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(string)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -164,7 +164,7 @@ BOOST_AUTO_TEST_CASE(format_bool_true_singleline)
bytes expectedBytes = toBigEndian(u256{true});
ABIType abiType{ABIType::Boolean, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "true", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(bool)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE(format_bool_false_singleline)
bytes expectedBytes = toBigEndian(u256{false});
ABIType abiType{ABIType::Boolean, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "false", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(bool)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -199,7 +199,7 @@ BOOST_AUTO_TEST_CASE(format_bool_left_singleline)
bytes expectedBytes = toBigEndian(u256{false});
ABIType abiType{ABIType::Boolean, ABIType::AlignLeft, 32};
Parameter param{expectedBytes, "left(false)", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(bool)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -214,7 +214,7 @@ BOOST_AUTO_TEST_CASE(format_hex_number_right_singleline)
bytes expectedBytes = result + bytes(32 - result.size(), 0);
ABIType abiType{ABIType::Hex, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "right(0x42)", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(bool)", {0}, arguments, expectations};
call.omitsArrow = false;
@ -228,7 +228,7 @@ BOOST_AUTO_TEST_CASE(format_empty_byte_range)
bytes expectedBytes;
ABIType abiType{ABIType::None, ABIType::AlignNone, 0};
Parameter param{expectedBytes, "1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}};
FunctionCallExpectations expectations{vector<Parameter>{param}, false, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{}, string{}};
FunctionCall call{"f()", {0}, arguments, expectations};
call.omitsArrow = false;
@ -242,7 +242,7 @@ BOOST_AUTO_TEST_CASE(format_failure_singleline)
bytes expectedBytes = toBigEndian(u256{1});
ABIType abiType{ABIType::UnsignedDec, ABIType::AlignRight, 32};
Parameter param{expectedBytes, "1", abiType, FormatInfo{}};
FunctionCallExpectations expectations{vector<Parameter>{}, true, string{}};
FunctionCallExpectations expectations{vector<Parameter>{}, true, string{}, {}};
FunctionCallArgs arguments{vector<Parameter>{param}, string{}};
FunctionCall call{"f(uint8)", {0}, arguments, expectations};
call.omitsArrow = false;