Adding debug info on gas costs when updating gas values.

Co-authored-by: Kamil Śliwak <kamil.sliwak@codepoets.it>

test/libsolidity/SemanticTest.cpp

@@ -294,7 +294,13 @@ TestCase::TestResult SemanticTest::runTest(
 		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
-			_stream << test.format(errorReporter, _linePrefix, false, _formatted, false) << endl;
+			_stream << test.format(
+				errorReporter,
+				_linePrefix,
+				TestFunctionCall::RenderMode::ExpectedValuesExpectedGas,
+				_formatted,
+				/* _interactivePrint */ true
+			) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		_stream << endl;
@@ -302,7 +308,13 @@ TestCase::TestResult SemanticTest::runTest(
 		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
-			_stream << test.format(errorReporter, _linePrefix, !m_gasCostFailure, _formatted, m_gasCostFailure) << endl;
+			_stream << test.format(
+				errorReporter,
+				_linePrefix,
+				m_gasCostFailure ? TestFunctionCall::RenderMode::ExpectedValuesActualGas : TestFunctionCall::RenderMode::ActualValuesExpectedGas,
+				_formatted,
+				/* _interactivePrint */ true
+			) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		AnsiColorized(_stream, _formatted, {BOLD, RED})
@@ -399,9 +411,8 @@ void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) con
 	for (TestFunctionCall const& test: m_tests)
 		_stream << test.format(
 			"",
-			/* _renderResult = */ !m_gasCostFailure,
+			m_gasCostFailure ? TestFunctionCall::RenderMode::ExpectedValuesActualGas : TestFunctionCall::RenderMode::ActualValuesExpectedGas,
-			/* _highlight = */ false,
+			/* _highlight = */ false
-			/* _renderGasCostResult */ m_gasCostFailure
 		) << endl;
 }
 

test/libsolidity/util/TestFunctionCall.cpp

@@ -35,9 +35,9 @@ using Token = soltest::Token;
 string TestFunctionCall::format(
 	ErrorReporter& _errorReporter,
 	string const& _linePrefix,
-	bool const _renderResult,
+	RenderMode _renderMode,
 	bool const _highlight,
-	bool const _renderGasCostResult
+	bool const _interactivePrint
 ) const
 {
 	stringstream stream;
@@ -66,9 +66,12 @@ string TestFunctionCall::format(
 			stream << _linePrefix << newline << ws << "storage" << colon << ws;
 			soltestAssert(m_rawBytes.size() == 1, "");
 			soltestAssert(m_call.expectations.rawBytes().size() == 1, "");
-			bool isEmpty = _renderResult ? m_rawBytes.front() == 0 : m_call.expectations.rawBytes().front() == 0;
+			bool isEmpty =
+				_renderMode == RenderMode::ActualValuesExpectedGas ?
+				m_rawBytes.front() == 0 :
+				m_call.expectations.rawBytes().front() == 0;
 			string output = isEmpty ? "empty" : "nonempty";
-			if (_renderResult && !matchesExpectation())
+			if (_renderMode == RenderMode::ActualValuesExpectedGas && !matchesExpectation())
 				AnsiColorized(stream, highlight, {util::formatting::RED_BACKGROUND}) << output;
 			else
 				stream << output;
@@ -105,7 +108,7 @@ string TestFunctionCall::format(
 
 			if (m_call.omitsArrow)
 			{
-				if (_renderResult && (m_failure || !matchesExpectation()))
+				if (_renderMode == RenderMode::ActualValuesExpectedGas && (m_failure || !matchesExpectation()))
 					stream << ws << arrow;
 			}
 			else
@@ -124,11 +127,11 @@ string TestFunctionCall::format(
 
 		/// Format either the expected output or the actual result output
 		string result;
-		if (!_renderResult)
+		if (_renderMode != RenderMode::ActualValuesExpectedGas)
 		{
 			bool const isFailure = m_call.expectations.failure;
 			result = isFailure ?
-				formatFailure(_errorReporter, m_call, m_rawBytes, _renderResult, highlight) :
+				formatFailure(_errorReporter, m_call, m_rawBytes, /* _renderResult */ false, highlight) :
 				formatRawParameters(m_call.expectations.result);
 			if (!result.empty())
 				AnsiColorized(stream, highlight, {util::formatting::RED_BACKGROUND}) << ws << result;
@@ -141,7 +144,7 @@ string TestFunctionCall::format(
 			bytes output = m_rawBytes;
 			bool const isFailure = m_failure;
 			result = isFailure ?
-				formatFailure(_errorReporter, m_call, output, _renderResult, highlight) :
+				formatFailure(_errorReporter, m_call, output, _renderMode == RenderMode::ActualValuesExpectedGas, highlight) :
 				matchesExpectation() ?
 					formatRawParameters(m_call.expectations.result) :
 					formatBytesParameters(
@@ -205,7 +208,7 @@ string TestFunctionCall::format(
 			}
 		}
 
-		stream << formatGasExpectations(_linePrefix, _renderGasCostResult);
+		stream << formatGasExpectations(_linePrefix, _renderMode == RenderMode::ExpectedValuesActualGas, _interactivePrint);
 	};
 
 	formatOutput(m_call.displayMode == FunctionCall::DisplayMode::SingleLine);
@@ -321,12 +324,35 @@ string TestFunctionCall::formatRawParameters(
 	return os.str();
 }
 
-string TestFunctionCall::formatGasExpectations(string const& _linePrefix, bool const _renderGasCostResult) const
+string TestFunctionCall::formatGasExpectations(
+	string const& _linePrefix,
+	bool _useActualCost,
+	bool _showDifference
+) const
 {
 	stringstream os;
-	for (auto const& [runType, gasUsed]: (_renderGasCostResult ? m_gasCosts : m_call.expectations.gasUsed))
+	for (auto const& [runType, gasUsed]: (_useActualCost ? m_gasCosts : m_call.expectations.gasUsed))
 		if (!runType.empty())
+		{
+			bool differentResults =
+				m_gasCosts.count(runType) > 0 &&
+				m_call.expectations.gasUsed.count(runType) > 0 &&
+				m_gasCosts.at(runType) != m_call.expectations.gasUsed.at(runType);
+
+			s256 difference = 0;
+			if (differentResults)
+				difference =
+					static_cast<s256>(m_gasCosts.at(runType)) -
+					static_cast<s256>(m_call.expectations.gasUsed.at(runType));
+			int percent = 0;
+			if (differentResults)
+				percent = static_cast<int>(
+					100.0 * (static_cast<double>(difference) / static_cast<double>(m_call.expectations.gasUsed.at(runType)))
+				);
 			os << endl << _linePrefix << "// gas " << runType << ": " << (gasUsed.str());
+			if (_showDifference && differentResults && _useActualCost)
+				os << " [" << showpos << difference << " (" << percent << "%)]";
+		}
 	return os.str();
 }
 

test/libsolidity/util/TestFunctionCall.h

@@ -42,37 +42,45 @@ namespace solidity::frontend::test
 class TestFunctionCall
 {
 public:
+	enum class RenderMode
+	{
+		ExpectedValuesExpectedGas,
+		ActualValuesExpectedGas,
+		ExpectedValuesActualGas
+	};
+
 	TestFunctionCall(FunctionCall _call): m_call(std::move(_call)), m_gasCosts(m_call.expectations.gasUsed) {}
 
 	/// Formats this function call test and applies the format that was detected during parsing.
-	/// If _renderResult is false, the expected result of the call will be used, if it's true
+	/// _renderMode determines the source of values to be inserted into the updated test expectations.
-	/// the actual result is used.
+	///     RenderMode::ActualValuesExpectedGas: use the values produced by the test but for gas preserve the original expectations,
+	///     RenderMode::ExpectedValuesExpectedGas: preserve the original expectations for both gas and other values,
+	///     RenderMode::ExpectedValuesActualGas: use the original expectations but for gas use actual values,
 	/// If _highlight is false, it's formatted without colorized highlighting. If it's true, AnsiColorized is
 	/// used to apply a colorized highlighting.
-	/// If __renderGasCostResult is false, the expected gas costs will be used, if it's true
+	/// If _interactivePrint is true, we are producing output that will be interactively shown to the user
-	/// the actual gas costs will be used
+	/// in the terminal rather than used to update the expectations in the test file.
 	/// If test expectations do not match, the contract ABI is consulted in order to get the
 	/// right encoding for returned bytes, based on the parsed return types.
 	/// Reports warnings and errors to the error reporter.
 	std::string format(
 		ErrorReporter& _errorReporter,
 		std::string const& _linePrefix = "",
-		bool const _renderResult = false,
+		RenderMode _renderMode = RenderMode::ExpectedValuesExpectedGas,
 		bool const _highlight = false,
-		bool const _renderGasCostResult = false
+		bool const _interactivePrint = false
 	) const;
 
 	/// Overloaded version that passes an error reporter which is never used outside
 	/// of this function.
 	std::string format(
 		std::string const& _linePrefix = "",
-		bool const _renderResult = false,
+		RenderMode const _renderMode = RenderMode::ExpectedValuesExpectedGas,
-		bool const _highlight = false,
+		bool const _highlight = false
-		bool const _renderGasCostResult = false
 	) const
 	{
 		ErrorReporter reporter;
-		return format(reporter, _linePrefix, _renderResult, _highlight, _renderGasCostResult);
+		return format(reporter, _linePrefix, _renderMode, _highlight);
 	}
 
 	/// Resets current results in case the function was called and the result
@@ -124,7 +132,8 @@ private:
 	/// Formats gas usage expectations one per line
 	std::string formatGasExpectations(
 		std::string const& _linePrefix,
-		bool const _renderGasCostResult
+		bool _useActualCost,
+		bool _showDifference
 	) const;
 
 	/// Compares raw expectations (which are converted to a byte representation before),

test/libsolidity/util/TestFunctionCallTests.cpp

@@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline)
 	test.setRawBytes(toBigEndian(u256{2}));
 	test.setFailure(false);
 
-	BOOST_REQUIRE_EQUAL(test.format("", true), "// f(uint8): 1 -> 2");
+	BOOST_REQUIRE_EQUAL(test.format("", TestFunctionCall::RenderMode::ActualValuesExpectedGas), "// f(uint8): 1 -> 2");
 }
 
 BOOST_AUTO_TEST_CASE(format_unsigned_singleline_signed_encoding)
@@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(format_unsigned_singleline_signed_encoding)
 	test.setRawBytes(toBigEndian(u256{-1}));
 	test.setFailure(false);
 
-	BOOST_REQUIRE_EQUAL(test.format("", true), "// f(uint8): 1 -> -1");
+	BOOST_REQUIRE_EQUAL(test.format("", TestFunctionCall::RenderMode::ActualValuesExpectedGas), "// f(uint8): 1 -> -1");
 }
 
 BOOST_AUTO_TEST_CASE(format_unsigned_multiline)
@@ -119,7 +119,7 @@ BOOST_AUTO_TEST_CASE(format_signed_singleline)
 	test.setRawBytes(toBigEndian(u256{-2}));
 	test.setFailure(false);
 
-	BOOST_REQUIRE_EQUAL(test.format("", true), "// f(int8): -1 -> -2");
+	BOOST_REQUIRE_EQUAL(test.format("", TestFunctionCall::RenderMode::ActualValuesExpectedGas), "// f(int8): -1 -> -2");
 }
 
 BOOST_AUTO_TEST_CASE(format_hex_singleline)
@@ -142,7 +142,7 @@ BOOST_AUTO_TEST_CASE(format_hex_singleline)
 	test.setRawBytes(actualBytes);
 	test.setFailure(false);
 
-	BOOST_REQUIRE_EQUAL(test.format("", true), "// f(bytes32): 0x31 -> 0x32");
+	BOOST_REQUIRE_EQUAL(test.format("", TestFunctionCall::RenderMode::ActualValuesExpectedGas), "// f(bytes32): 0x31 -> 0x32");
 }
 
 BOOST_AUTO_TEST_CASE(format_hex_string_singleline)
@@ -177,7 +177,7 @@ BOOST_AUTO_TEST_CASE(format_bool_true_singleline)
 	test.setRawBytes(actualBytes);
 	test.setFailure(false);
 
-	BOOST_REQUIRE_EQUAL(test.format("", true), "// f(bool): true -> false");
+	BOOST_REQUIRE_EQUAL(test.format("", TestFunctionCall::RenderMode::ActualValuesExpectedGas), "// f(bool): true -> false");
 }
 
 BOOST_AUTO_TEST_CASE(format_bool_false_singleline)