Merge pull request #7095 from ethereum/isoltest-failure-reason

[isoltest] Support FAILURE with reason

test/libsolidity/semanticTests/smoke/alignment.sol

@@ -26,3 +26,4 @@ contract D {
 // stateBytes() -> left(0x4200ef)
 // internalStateDecimal() -> 0x20
 // update(bool,uint256,bytes32): false, -23, left(0x2300ef) -> false, -23, left(0x2300ef)
+

test/libsolidity/semanticTests/smoke/arrays.sol

@@ -0,0 +1,44 @@
+pragma experimental ABIEncoderV2;
+
+contract C {
+    struct T {
+        uint a;
+        uint b;
+        string s;
+    }
+    bool[2][] flags;
+    function r() public returns (bool[3] memory) {
+        return [true, false, true];
+    }
+    function s() public returns (uint[2] memory, uint) {
+        return ([uint(123), 456], 789);
+    }
+    function u() public returns (T[2] memory) {
+        return [T(23, 42, "any"), T(555, 666, "any")];
+    }
+    function v() public returns (bool[2][] memory) {
+        return flags;
+    }
+    function w1() public returns (string[1] memory) {
+        return ["any"];
+    }
+    function w2() public returns (string[2] memory) {
+        return ["any", "any"];
+    }
+    function w3() public returns (string[3] memory) {
+        return ["any", "any", "any"];
+    }
+    function x() public returns (string[2] memory, string[3] memory) {
+        return (["any", "any"], ["any", "any", "any"]);
+    }
+}
+// ----
+// r() -> true, false, true
+// s() -> 123, 456, 789
+// u() -> 0x20, 0x40, 0xE0, 23, 42, 0x60, 3, "any", 555, 666, 0x60, 3, "any"
+// v() -> 0x20, 0
+// w1() -> 0x20, 0x20, 3, "any"
+// w2() -> 0x20, 0x40, 0x80, 3, "any", 3, "any"
+// w3() -> 0x20, 0x60, 0xa0, 0xe0, 3, "any", 3, "any", 3, "any"
+// x() -> 0x40, 0x0100, 0x40, 0x80, 3, "any", 3, "any", 0x60, 0xa0, 0xe0, 3, "any", 3, "any", 3, "any"
+

test/libsolidity/semanticTests/smoke/basic.sol

@@ -0,0 +1,40 @@
+pragma experimental ABIEncoderV2;
+
+contract C {
+    function d() public {
+    }
+    function e() public payable returns (uint) {
+        return msg.value;
+    }
+    function f(uint a) public pure returns (uint, uint) {
+        return (a, a);
+    }
+    function g() public  pure returns (uint, uint) {
+        return (2, 3);
+    }
+    function h(uint x, uint y) public  pure returns (uint) {
+        return x - y;
+    }
+    function i(bool b) public  pure returns (bool) {
+        return !b;
+    }
+    function j(bytes32 b) public pure returns (bytes32, bytes32) {
+        return (b, b);
+    }
+    function k() public pure returns (uint) {
+        return msg.data.length;
+    }
+    function l(uint a) public pure returns (uint d) {
+        return a * 7;
+    }
+}
+// ----
+// d() ->
+// e(), 1 ether -> 1
+// f(uint256): 3 -> 3, 3
+// g() -> 2, 3
+// h(uint256,uint256): 1, -2 -> 3
+// i(bool): true -> false
+// j(bytes32): 0x10001 -> 0x10001, 0x10001
+// k(): hex"4200efef" -> 8
+// l(uint256): 99 -> 693

test/libsolidity/semanticTests/smoke/bytes_and_strings.sol

@@ -0,0 +1,22 @@
+contract C {
+    function e(bytes memory b) public pure returns (bytes memory) {
+        return b;
+    }
+    function f() public pure returns (string memory, string memory) {
+        return ("any", "any");
+    }
+    function g() public pure returns (string memory, uint, string memory) {
+        return ("any", 42, "any");
+    }
+    function h() public pure returns (string memory) {
+        return "any";
+    }
+}
+// ----
+// e(bytes): 32, 3, hex"AB33BB" -> 32, 3, left(0xAB33BB)
+// e(bytes): 32, 32, 0x20 -> 32, 32, 0x20
+// e(bytes): 32, 3, hex"AB33FF" -> 32, 3, hex"ab33ff0000000000000000000000000000000000000000000000000000000000"
+// f() -> 0x40, 0x80, 3, "any", 3, "any"
+// g() -> 0x60, 0x2a, 0xa0, 3, "any", 3, "any"
+// h() -> 0x20, 3, "any"
+

test/libsolidity/semanticTests/smoke/constructor.sol

@@ -0,0 +1,18 @@
+contract C {
+    uint public state = 0;
+    constructor(uint _state) public payable {
+        state = _state;
+    }
+    function balance() public payable returns (uint256) {
+        return address(this).balance;
+    }
+    function update(uint _state) public {
+        state = _state;
+    }
+}
+// ----
+// constructor(), 2 ether: 3 ->
+// state() -> 3
+// balance() -> 2
+// update(uint256): 4
+// state() -> 4

test/libsolidity/semanticTests/smoke/failure.sol

@@ -0,0 +1,10 @@
+contract C {
+    function e() public {
+        revert("Transaction failed.");
+    }
+}
+// ====
+// EVMVersion: >homestead
+// ----
+// _() -> FAILURE
+// e() -> FAILURE, hex"08c379a0", 0x20, 19, "Transaction failed."

test/libsolidity/semanticTests/smoke/multiline.sol

@@ -11,3 +11,4 @@ contract C {
 // g()
 // # g() does not exist #
 // -> FAILURE
+

test/libsolidity/semanticTests/smoke/multiline_comments.sol

@@ -17,3 +17,4 @@ contract C {
 // 1
 // -> 5
 // # Should return sum of all parameters. #
+

test/libsolidity/semanticTests/smoke/structs.sol

@@ -0,0 +1,22 @@
+pragma experimental ABIEncoderV2;
+
+contract C {
+    struct S {
+        uint a;
+        uint b;
+    }
+    struct T {
+        uint a;
+        uint b;
+        string s;
+    }
+    function s() public returns (S memory) {
+        return S(23, 42);
+    }
+    function t() public returns (T memory) {
+        return T(23, 42, "any");
+    }
+}
+// ----
+// s() -> 23, 42
+// t() -> 0x20, 23, 42, 0x60, 3, "any"

test/libsolidity/semanticTests/smoke_test.sol

@@ -1,121 +0,0 @@
-pragma experimental ABIEncoderV2;
-
-contract C {
-    struct S {
-        uint a;
-        uint b;
-    }
-    struct T {
-        uint a;
-        uint b;
-        string s;
-    }
-    uint public state = 0;
-    bool[2][] flags;
-    constructor(uint _state) public payable {
-        state = _state;
-    }
-    function balance() payable public returns (uint256) {
-        return address(this).balance;
-    }
-    function e(uint a) public {
-        state = a;
-    }
-    function f() payable public returns (uint) {
-        return 2;
-    }
-    function f(uint a) public returns (uint, uint) {
-        return (a, a);
-    }
-    function g() public returns (uint, uint) {
-        return (2, 3);
-    }
-    function h(uint x, uint y) public returns (uint) {
-        return x - y;
-    }
-    function j(bool b) public returns (bool) {
-        return !b;
-    }
-    function k(bytes32 b) public returns (bytes32, bytes32) {
-        return (b, b);
-    }
-    function l() public returns (uint256) {
-        return msg.data.length;
-    }
-    function m(bytes memory b) public returns (bytes memory) {
-        return b;
-    }
-    function n() public returns (string memory) {
-        return "any";
-    }
-    function o() public returns (string memory, string memory) {
-        return ("any", "any");
-    }
-    function p() public returns (string memory, uint, string memory) {
-        return ("any", 42, "any");
-    }
-    function q(uint a) public returns (uint d) {
-        return a * 7;
-    }
-    function r() public returns (bool[3] memory) {
-        return [true, false, true];
-    }
-    function s() public returns (uint[2] memory, uint) {
-        return ([uint(123), 456], 789);
-    }
-    function t1() public returns (S memory) {
-        return S(23, 42);
-    }
-    function t2() public returns (T memory) {
-        return T(23, 42, "any");
-    }
-    function u() public returns (T[2] memory) {
-        return [T(23, 42, "any"), T(555, 666, "any")];
-    }
-    function v() public returns (bool[2][] memory) {
-        return flags;
-    }
-    function w1() public returns (string[1] memory) {
-        return ["any"];
-    }
-    function w2() public returns (string[2] memory) {
-        return ["any", "any"];
-    }
-    function w3() public returns (string[3] memory) {
-        return ["any", "any", "any"];
-    }
-    function x() public returns (string[2] memory, string[3] memory) {
-        return (["any", "any"], ["any", "any", "any"]);
-    }
-}
-// ----
-// constructor(), 2 ether: 3 ->
-// state() -> 3
-// balance() -> 2
-// _() -> FAILURE
-// e(uint256): 4
-// f() -> 2
-// f(uint256): 3 -> 3, 3
-// f(), 1 ether -> 2
-// g() -> 2, 3
-// g1() -> FAILURE
-// h(uint256,uint256): 1, -2 -> 3
-// j(bool): true -> false
-// k(bytes32): 0x10001 -> 0x10001, 0x10001
-// l(): hex"4200efef" -> 8
-// m(bytes): 32, 32, 0x20 -> 32, 32, 0x20
-// m(bytes): 32, 3, hex"AB33BB" -> 32, 3, left(0xAB33BB)
-// m(bytes): 32, 3, hex"AB33FF" -> 32, 3, hex"ab33ff0000000000000000000000000000000000000000000000000000000000"
-// n() -> 0x20, 3, "any"
-// o() -> 0x40, 0x80, 3, "any", 3, "any"
-// p() -> 0x60, 0x2a, 0xa0, 3, "any", 3, "any"
-// q(uint256): 99 -> 693
-// r() -> true, false, true
-// s() -> 123, 456, 789
-// t1() -> 23, 42
-// t2() -> 0x20, 23, 42, 0x60, 3, "any"
-// v() -> 32, 0
-// w1() -> 0x20, 0x20, 3, "any"
-// w2() -> 0x20, 0x40, 0x80, 3, "any", 3, "any"
-// w3() -> 0x20, 0x60, 0xa0, 0xe0, 3, "any", 3, "any", 3, "any"
-// x() -> 0x40, 0x0100, 0x40, 0x80, 3, "any", 3, "any", 0x60, 0xa0, 0xe0, 3, "any", 3, "any", 3, "any"

test/libsolidity/util/BytesUtils.cpp

@@ -17,6 +17,7 @@
 
 #include <test/libsolidity/util/BytesUtils.h>
 
+#include <test/libsolidity/util/ContractABIUtils.h>
 #include <test/libsolidity/util/SoltestErrors.h>
 
 #include <liblangutil/Common.h>
@@ -202,22 +203,25 @@ string BytesUtils::formatString(bytes const& _bytes, size_t _cutOff)
 	return os.str();
 }
 
-string BytesUtils::formatRawBytes(bytes const& _bytes)
+string BytesUtils::formatRawBytes(
+	bytes const& _bytes,
+	dev::solidity::test::ParameterList const& _parameters,
+	string _linePrefix)
 {
-	if (_bytes.empty())
+	soltestAssert(_bytes.size() == ContractABIUtils::encodingSize(_parameters), "");
-		return "[]";
 
 	stringstream os;
 	auto it = _bytes.begin();
-	for (size_t i = 0; i < _bytes.size(); i += 32)
+
+	for (auto const& parameter: _parameters)
 	{
-		bytes byteRange{it, it + 32};
+		bytes byteRange{it, it + static_cast<long>(parameter.abiType.size)};
 
-		os << "  " << byteRange;
+		os << _linePrefix << byteRange;
-
+		if (&parameter != &_parameters.back())
-		it += 32;
-		if (it != _bytes.end())
 			os << endl;
+
+		it += static_cast<long>(parameter.abiType.size);
 	}
 
 	return os.str();
@@ -271,6 +275,8 @@ string BytesUtils::formatBytesRange(
 	bool _highlight
 )
 {
+	soltestAssert(_bytes.size() == ContractABIUtils::encodingSize(_parameters), "");
+
 	stringstream os;
 	auto it = _bytes.begin();
 
@@ -287,11 +293,12 @@ string BytesUtils::formatBytesRange(
 		else
 			os << parameter.rawString;
 
-
-		it += static_cast<long>(parameter.abiType.size);
 		if (&parameter != &_parameters.back())
 			os << ", ";
+
+		it += static_cast<long>(parameter.abiType.size);
 	}
+
 	return os.str();
 }
 

test/libsolidity/util/BytesUtils.h

@@ -101,10 +101,14 @@ public:
 		return formatString(_bytes, _bytes.size());
 	}
 
-	/// Returns a string representation of given _bytes. Adds a newline
-	/// every 32 bytes to increase readability.
 	/// Used to print returned bytes from function calls to the commandline.
-	static std::string formatRawBytes(bytes const& _bytes);
+	/// Returns a string representation of given _bytes in ranges of 32 bytes.
+	/// If _withSignature is true, the first 4 bytes will be formatted separately.
+	static std::string formatRawBytes(
+		bytes const& _bytes,
+		ParameterList const& _parameters,
+		std::string _linePrefix = ""
+	);
 
 	/// Formats given _bytes with type information passed in _abiType.
 	static std::string formatBytes(bytes const& _bytes, ABIType const& _abiType);

test/libsolidity/util/ContractABIUtils.cpp

@@ -289,16 +289,47 @@ dev::solidity::test::ParameterList ContractABIUtils::preferredParameters(
 {
 	if (_targetParameters.size() != _sourceParameters.size())
 	{
-		auto sizeFold = [](size_t const _a, Parameter const& _b) { return _a + _b.abiType.size; };
-		size_t encodingSize = accumulate(_targetParameters.begin(), _targetParameters.end(), size_t{0}, sizeFold);
-
 		_errorReporter.warning(
 			"Encoding does not match byte range. The call returned " +
 			to_string(_bytes.size()) + " bytes, but " +
-			to_string(encodingSize) + " bytes were expected."
+			to_string(encodingSize(_targetParameters)) + " bytes were expected."
 		);
 		return _sourceParameters;
 	}
 	else
 		return _targetParameters;
 }
+
+dev::solidity::test::ParameterList ContractABIUtils::defaultParameters(size_t count)
+{
+	ParameterList parameters;
+
+	fill_n(
+		back_inserter(parameters),
+		count,
+		Parameter{bytes(), "", ABIType{ABIType::UnsignedDec}, FormatInfo{}}
+	);
+
+	return parameters;
+}
+
+dev::solidity::test::ParameterList ContractABIUtils::failureParameters()
+{
+	ParameterList parameters;
+
+	parameters.push_back(Parameter{bytes(), "", ABIType{ABIType::HexString, ABIType::AlignNone, 4}, FormatInfo{}});
+	parameters.push_back(Parameter{bytes(), "", ABIType{ABIType::Hex}, FormatInfo{}});
+	parameters.push_back(Parameter{bytes(), "", ABIType{ABIType::UnsignedDec}, FormatInfo{}});
+	parameters.push_back(Parameter{bytes(), "", ABIType{ABIType::String}, FormatInfo{}});
+
+	return parameters;
+}
+
+size_t ContractABIUtils::encodingSize(
+	dev::solidity::test::ParameterList const& _parameters
+)
+{
+	auto sizeFold = [](size_t const _a, Parameter const& _b) { return _a + _b.abiType.size; };
+
+	return accumulate(_parameters.begin(), _parameters.end(), size_t{0}, sizeFold);
+}

test/libsolidity/util/ContractABIUtils.h

@@ -65,6 +65,18 @@ public:
 		bytes const& _bytes
 	);
 
+	/// Returns a list of parameters corresponding to the encoding of
+	/// returned values in case of a failure.
+	static ParameterList failureParameters();
+
+	/// Returns _count parameters with their type set to ABIType::UnsignedDec
+	/// and their size set to 32 bytes.
+	static ParameterList defaultParameters(size_t count = 0);
+
+	/// Calculates the encoding size of given _parameters based
+	/// on the size of their types.
+	static size_t encodingSize(ParameterList const& _paremeters);
+
 private:
 	/// Parses and translates a single type and returns a list of
 	/// internal type representations of isoltest.

test/libsolidity/util/TestFileParserTests.cpp

@@ -219,6 +219,23 @@ BOOST_AUTO_TEST_CASE(non_existent_call_revert)
 	testFunctionCall(calls.at(0), Mode::MultiLine, "i_am_not_there()", true);
 }
 
+BOOST_AUTO_TEST_CASE(call_revert_message)
+{
+	char const* source = R"(
+		// f() -> FAILURE, hex"08c379a0", 0x20, 6, "Revert"
+	)";
+	auto const calls = parse(source);
+	BOOST_REQUIRE_EQUAL(calls.size(), 1);
+	testFunctionCall(
+		calls.at(0),
+		Mode::SingleLine,
+		"f()",
+		true,
+		fmt::encodeArgs(),
+		fromHex("08c379a0") + fmt::encodeDyn(string{"Revert"})
+	);
+}
+
 BOOST_AUTO_TEST_CASE(call_expectations_empty_single_line)
 {
 	char const* source = R"(

test/libsolidity/util/TestFunctionCall.cpp

@@ -101,7 +101,7 @@ string TestFunctionCall::format(
 		{
 			bool const isFailure = m_call.expectations.failure;
 			result = isFailure ?
-				failure :
+				formatFailure(_errorReporter, m_call, m_rawBytes, _renderResult, highlight) :
 				formatRawParameters(m_call.expectations.result);
 			if (!result.empty())
 				AnsiColorized(stream, highlight, {dev::formatting::RED_BACKGROUND}) << ws << result;
@@ -111,7 +111,7 @@ string TestFunctionCall::format(
 			bytes output = m_rawBytes;
 			bool const isFailure = m_failure;
 			result = isFailure ?
-				failure :
+				formatFailure(_errorReporter, m_call, output, _renderResult, highlight) :
 				matchesExpectation() ?
 					formatRawParameters(m_call.expectations.result) :
 					formatBytesParameters(
@@ -122,6 +122,29 @@ string TestFunctionCall::format(
 						highlight
 					);
 
+			if (!matchesExpectation())
+			{
+				boost::optional<ParameterList> abiParams;
+
+				if (isFailure && !output.empty())
+					abiParams = boost::make_optional(ContractABIUtils::failureParameters());
+				else
+					abiParams = ContractABIUtils::parametersFromJsonOutputs(
+						_errorReporter,
+						m_contractABI,
+						m_call.signature
+					);
+
+				string bytesOutput = abiParams ?
+					BytesUtils::formatRawBytes(output, abiParams.get(), _linePrefix) :
+					_linePrefix + "[]";
+
+				_errorReporter.warning(
+					"The call to \"" + m_call.signature + "\" returned \n" +
+					bytesOutput
+				);
+			}
+
 			if (isFailure)
 				AnsiColorized(stream, highlight, {dev::formatting::RED_BACKGROUND}) << ws << result;
 			else
@@ -155,49 +178,90 @@ string TestFunctionCall::formatBytesParameters(
 	bytes const& _bytes,
 	string const& _signature,
 	dev::solidity::test::ParameterList const& _parameters,
-	bool _highlight
+	bool _highlight,
+	bool _failure
 ) const
 {
 	using ParameterList = dev::solidity::test::ParameterList;
 
 	stringstream os;
+
 	if (_bytes.empty())
 		return {};
 
-	_errorReporter.warning("The call to \"" + _signature + "\" returned \n" + BytesUtils::formatRawBytes(_bytes));
+	if (_failure)
-
-	boost::optional<ParameterList> abiParams = ContractABIUtils::parametersFromJsonOutputs(
-		_errorReporter,
-		m_contractABI,
-		_signature
-	);
-
-	if (abiParams)
 	{
-		boost::optional<ParameterList> preferredParams = ContractABIUtils::preferredParameters(
+		os << BytesUtils::formatBytesRange(
-			_errorReporter,
+			_bytes,
-			_parameters,
+			ContractABIUtils::failureParameters(),
-			abiParams.get(),
+			_highlight
-			_bytes
 		);
 
-		if (preferredParams)
+		return os.str();
-		{
-			ContractABIUtils::overwriteParameters(_errorReporter, preferredParams.get(), abiParams.get());
-			os << BytesUtils::formatBytesRange(_bytes, preferredParams.get(), _highlight);
-		}
 	}
 	else
 	{
-		ParameterList defaultParameters;
+		boost::optional<ParameterList> abiParams = ContractABIUtils::parametersFromJsonOutputs(
-		fill_n(
+			_errorReporter,
-			back_inserter(defaultParameters),
+			m_contractABI,
-			ceil(_bytes.size() / 32),
+			_signature
-			Parameter{bytes(), "", ABIType{ABIType::Hex}, FormatInfo{}}
 		);
-		ContractABIUtils::overwriteParameters(_errorReporter, defaultParameters, _parameters);
+
-		os << BytesUtils::formatBytesRange(_bytes, defaultParameters, _highlight);
+		if (abiParams)
+		{
+			boost::optional<ParameterList> preferredParams = ContractABIUtils::preferredParameters(
+				_errorReporter,
+				_parameters,
+				abiParams.get(),
+				_bytes
+			);
+
+			if (preferredParams)
+			{
+				ContractABIUtils::overwriteParameters(_errorReporter, preferredParams.get(), abiParams.get());
+				os << BytesUtils::formatBytesRange(_bytes, preferredParams.get(), _highlight);
+			}
+		}
+		else
+		{
+			ParameterList defaultParameters = ContractABIUtils::defaultParameters(ceil(_bytes.size() / 32));
+
+			ContractABIUtils::overwriteParameters(_errorReporter, defaultParameters, _parameters);
+			os << BytesUtils::formatBytesRange(_bytes, defaultParameters, _highlight);
+		}
+		return os.str();
 	}
+}
+
+string TestFunctionCall::formatFailure(
+	ErrorReporter& _errorReporter,
+	dev::solidity::test::FunctionCall const& _call,
+	bytes const& _output,
+	bool _renderResult,
+	bool _highlight
+) const
+{
+	using Token = soltest::Token;
+
+	stringstream os;
+
+	os << formatToken(Token::Failure);
+
+	if (!_output.empty())
+		os << ", ";
+
+	if (_renderResult)
+		os << formatBytesParameters(
+			_errorReporter,
+			_output,
+			_call.signature,
+			_call.expectations.result,
+			_highlight,
+			true
+		);
+	else
+		os << formatRawParameters(_call.expectations.result);
+
 	return os.str();
 }
 

test/libsolidity/util/TestFunctionCall.h

@@ -95,7 +95,8 @@ private:
 		bytes const& _bytes,
 		std::string const& _signature,
 		ParameterList const& _params,
-		bool highlight = false
+		bool highlight = false,
+		bool failure = false
 	) const;
 
 	/// Formats a given _bytes applying the _abiType.
@@ -104,6 +105,15 @@ private:
 		ABIType const& _abiType
 	) const;
 
+	/// Formats a FAILURE plus additional parameters, if e.g. a revert message was returned.
+	std::string formatFailure(
+		ErrorReporter& _errorReporter,
+		FunctionCall const& _call,
+		bytes const& _output,
+		bool _renderResult,
+		bool _highlight
+	) const;
+
 	/// Formats the given parameters using their raw string representation.
 	std::string formatRawParameters(
 		ParameterList const& _params,